{
 "metadata": {
  "anaconda-cloud": {},
  "coursera": {
   "course_slug": "deep-neural-network",
   "graded_item_id": "BFd89",
   "launcher_item_id": "AH2rK"
  },
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "name": "",
  "signature": "sha256:45cdc14c1a335827db1260e8e3c7372d38bff0023a3332686edc52996a4a6546"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# TensorFlow Tutorial\n",
      "\n",
      "\u6b22\u8fce\u6765\u5230\u672c\u5468\u7684\u7f16\u7a0b\u4efb\u52a1\u3002\u5230\u76ee\u524d\u4e3a\u6b62\uff0c\u4f60\u4e00\u76f4\u4f7f\u7528numpy\u6765\u5efa\u7acb\u795e\u7ecf\u7f51\u7edc\u3002\u73b0\u5728\u6211\u4eec\u5c06\u5f15\u5bfc\u60a8\u6df1\u5165\u5b66\u4e60\u6846\u67b6\uff0c\u8ba9\u60a8\u66f4\u5bb9\u6613\u5730\u5efa\u7acb\u795e\u7ecf\u7f51\u7edc\u3002TensorFlow\uff0cPaddlePaddle\uff0cTorch\uff0cCaffe\uff0cKeras\u7b49\u673a\u5668\u5b66\u4e60\u6846\u67b6\u53ef\u4ee5\u663e\u7740\u52a0\u901f\u60a8\u7684\u673a\u5668\u5b66\u4e60\u5f00\u53d1\u3002\u8fd9\u4e9b\u6846\u67b6\u6709\u5f88\u591a\u6587\u6863\uff0c\u4f60\u53ef\u4ee5\u968f\u610f\u9605\u8bfb\u3002\u5728\u8fd9\u4e2a\u4efb\u52a1\u4e2d\uff0c\u60a8\u5c06\u5b66\u4e60\u5982\u4f55\u5728TensorFlow\u4e2d\u6267\u884c\u4ee5\u4e0b\u64cd\u4f5c\uff1a\n",
      "\n",
      "- \u521d\u59cb\u5316\u53d8\u91cf\n",
      "- \u5f00\u59cb\u4f60\u81ea\u5df1\u7684\u4f1a\u8bdd\n",
      "- \u8bad\u7ec3\u7b97\u6cd5\n",
      "- \u5b9e\u73b0\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc\n",
      "\n",
      "\u7f16\u7a0b\u6846\u67b6\u4e0d\u4ec5\u53ef\u4ee5\u7f29\u77ed\u60a8\u7684\u7f16\u7801\u65f6\u95f4\uff0c\u4f46\u6709\u65f6\u4e5f\u53ef\u4ee5\u6267\u884c\u4f18\u5316\u6765\u52a0\u901f\u60a8\u7684\u4ee3\u7801\u3002\n",
      "\n",
      "\n",
      "## 1 - Exploring the Tensorflow Library\n",
      "\n",
      "\u9996\u5148\u5bfc\u5165\u76f8\u5173\u5e93\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import math\n",
      "import numpy as np\n",
      "import h5py\n",
      "import matplotlib.pyplot as plt\n",
      "import tensorflow as tf\n",
      "from tensorflow.python.framework import ops\n",
      "from tf_utils import load_dataset, random_mini_batches, convert_to_one_hot, predict\n",
      "\n",
      "%matplotlib inline\n",
      "np.random.seed(1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u6211\u4eec\u5c06\u4ece\u4e00\u4e2a\u4f8b\u5b50\u5f00\u59cb\uff0c\u8fd9\u4e2a\u4f8b\u5b50\u91cc\u9700\u8981\u8ba1\u7b97\u4e00\u4e2a\u8bad\u7ec3\u6837\u4f8b\u7684\u635f\u5931\u3002\n",
      "$$loss = \\mathcal{L}(\\hat{y}, y) = (\\hat y^{(i)} - y^{(i)})^2 \\tag{1}$$"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "y_hat = tf.constant(36, name='y_hat')            # Define y_hat constant. Set to 36.\n",
      "y = tf.constant(39, name='y')                    # Define y. Set to 39\n",
      "\n",
      "loss = tf.Variable((y - y_hat)**2, name='loss')  # Create a variable for the loss\n",
      "\n",
      "init = tf.global_variables_initializer()         # When init is run later (session.run(init)),\n",
      "                                                 # the loss variable will be initialized and ready to be computed\n",
      "with tf.Session() as session:                    # Create a session and print the output\n",
      "    session.run(init)                            # Initializes the variables\n",
      "    print(session.run(loss))                     # Prints the loss"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "9\n"
       ]
      }
     ],
     "prompt_number": 38
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u7528TensorFlow\u7f16\u5199\u548c\u8fd0\u884c\u7a0b\u5e8f\u901a\u5e38\u6709\u4ee5\u4e0b\u6b65\u9aa4\uff1a\n",
      "\n",
      "1. \u521b\u5efa\u5c1a\u672a\u6267\u884c/\u8bc4\u4f30\u7684\u5f20\u91cf\uff08\u53d8\u91cf\uff09\n",
      "2. \u5728\u8fd9\u4e9b\u5f20\u91cf\u4e4b\u95f4\u5199\u5165\u64cd\u4f5c\u3002\n",
      "3. \u521d\u59cb\u5316\u60a8\u7684\u5f20\u91cf\u3002\n",
      "4. \u521b\u5efa\u4e00\u4e2a\u4f1a\u8bdd\u3002\n",
      "5. \u8fd0\u884c\u4f1a\u8bdd\u3002\u8fd9\u5c06\u8fd0\u884c\u4f60\u4e0a\u9762\u5df2\u7ecf\u5b8c\u6210\u7684\u5199\u64cd\u4f5c\u3002 \n",
      "\n",
      "Therefore, when we created a variable for the loss, we simply defined the loss as a function of other quantities, but did not evaluate its value. To evaluate it, we had to run `init=tf.global_variables_initializer()`. That initialized the loss variable, and in the last line we were finally able to evaluate the value of `loss` and print its value.\n",
      "\n",
      "\u73b0\u5728\u8ba9\u6211\u4eec\u770b\u4e00\u4e2a\u7b80\u5355\u7684\u4f8b\u5b50\u3002\u8fd0\u884c\u4e0b\u9762\u7684\u5355\u5143\u683c\uff1a"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "a = tf.constant(2)\n",
      "b = tf.constant(10)\n",
      "c = tf.multiply(a,b)\n",
      "print(c)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Tensor(\"Mul:0\", shape=(), dtype=int32)\n"
       ]
      }
     ],
     "prompt_number": 39
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u6b63\u5982\u6240\u6599\uff0c\u4f60\u4e0d\u4f1a\u770b\u523020\uff01\u4f60\u5f97\u5230\u662f\u4e00\u4e2a\u7c7b\u578b\u4e3a\u201cint32\u201d\u4e14\u6ca1\u6709shape\u5c5e\u6027\u7684\u5f20\u91cf\u3002\u4f60\u76ee\u524d\u6240\u505a\u7684\u53ea\u662f\u628a\u5b83\u653e\u5230\u201c\u8ba1\u7b97\u56fe\u201d\u4e2d\uff0c\u4f60\u8fd8\u6ca1\u6709\u8fd0\u884c\u8fd9\u4e2a\u8ba1\u7b97\u3002\u4e3a\u4e86\u8ba9\u8fd9\u4e24\u4e2a\u6570\u5b9e\u9645\u4e0a\u76f8\u4e58\uff0c\u4f60\u9700\u8981\u53bb\u521b\u5efa\u4e00\u4e2a\u4f1a\u8bdd\u5e76\u8fd0\u884c\u5b83."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sess = tf.Session()\n",
      "print(sess.run(c))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "20\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "Great! \u603b\u7ed3\u4e00\u4e0b, **\uff0c\u8bb0\u5f97\u521d\u59cb\u5316\u4f60\u7684\u53d8\u91cf\uff0c\u521b\u5efa\u4e00\u4e2a\u4f1a\u8bdd\u5e76\u8fd0\u884c\u4f1a\u8bdd\u4e2d\u7684\u64cd\u4f5c**. \n",
      "\n",
      "\u63a5\u4e0b\u6765\uff0c\u60a8\u8fd8\u5fc5\u987b\u4e86\u89e3\u5360\u4f4d\u7b26.\u5360\u4f4d\u7b26\u662f\u4e00\u4e2a\u60a8\u53ef\u4ee5\u7a0d\u540e\u6307\u5b9a\u503c\u7684\u5bf9\u8c61. \n",
      "\u8981\u4e3a\u5360\u4f4d\u7b26\u6307\u5b9a\u503c\uff0c\u53ef\u4ee5\u4f7f\u7528 \"feed dictionary\" (`feed_dict` variable)\u4f20\u503c\u3002 \u4e0b\u9762\uff0c\u6211\u4eec\u4e3ax\u521b\u5efa\u4e86\u4e00\u4e2a\u5360\u4f4d\u7b26\u3002\u8fd9\u5141\u8bb8\u6211\u4eec\u7a0d\u540e\u5728\u8fd0\u884c\u4f1a\u8bdd\u65f6\u4f20\u5165\u4e00\u4e2a\u6570\u5b57\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Change the value of x in the feed_dict\n",
      "\n",
      "x = tf.placeholder(tf.int64, name = 'x')\n",
      "print(sess.run(2 * x, feed_dict = {x: 3}))\n",
      "sess.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "6\n"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u5f53\u4f60\u7b2c\u4e00\u6b21\u5b9a\u4e49x\u4f60\u4e0d\u5fc5\u4e3a\u5b83\u6307\u5b9a\u4e00\u4e2a\u503c\u3002\u5360\u4f4d\u7b26\u53ea\u662f\u4e00\u4e2a\u53d8\u91cf\uff0c\u60a8\u5c06\u5728\u7a0d\u540e\u8fd0\u884c\u4f1a\u8bdd\u65f6\u5c06\u6570\u636e\u5206\u914d\u7ed9\u8be5\u53d8\u91cf\u3002\n",
      "\n",
      "\u4ee5\u4e0b\u662f\u53d1\u751f\u4e86\u7684\u4e8b\u60c5\uff1a\u5f53\u60a8\u6307\u5b9a\u8ba1\u7b97\u6240\u9700\u7684\u64cd\u4f5c\u65f6\uff0c\u60a8\u6b63\u5728\u544a\u8bc9TensorFlow\u5982\u4f55\u6784\u5efa\u8ba1\u7b97\u56fe\u3002\u8ba1\u7b97\u56fe\u53ef\u4ee5\u6709\u4e00\u4e9b\u5360\u4f4d\u7b26\uff0c\u5176\u503c\u5c06\u5728\u7a0d\u540e\u6307\u5b9a\u3002\u6700\u540e\uff0c\u5f53\u4f60\u8fd0\u884c\u4f1a\u8bdd\u65f6\uff0c\u4f60\u6b63\u5728\u544a\u8bc9TensorFlow\u6267\u884c\u8ba1\u7b97\u56fe\u3002"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 1.1 - Linear function\n",
      "\n",
      "\u8ba9\u6211\u4eec\u901a\u8fc7\u8ba1\u7b97\u4e0b\u9762\u7684\u7b49\u5f0f\u6765\u5f00\u59cb\u8fd9\u4e2a\u7f16\u7a0b\u7ec3\u4e60: $Y = WX + b$, where $W$ and $X$ are random matrices and b is a random vector. \n",
      "\n",
      "**Exercise**: \u8ba1\u7b97 $WX + b$\uff0c $W, X$, \u548c $b$ \u662f\u4ece\u968f\u673a\u6b63\u592a\u5206\u5e03\u4e2d\u62bd\u53d6\u7684. W is of shape (4, 3), X is (3,1) and b is (4,1). \u4e0b\u9762\u662f\u4e00\u4e2a\u5c0f\u4f8b\u5b50\uff0c\u544a\u8bc9\u4f60\u5982\u4f55\u5b9a\u4e49\u4e00\u4e2a\u5177\u6709shape\uff083,1\uff09\u7684\u5e38\u91cfX:\n",
      "```python\n",
      "X = tf.constant(np.random.randn(3,1), name = \"X\")\n",
      "\n",
      "```\n",
      "\u4e0b\u9762\u7684\u65b9\u6cd5\u53ef\u80fd\u5bf9\u4f60\u6709\u7528: \n",
      "- tf.matmul(..., ...) to do a matrix multiplication\n",
      "- tf.add(..., ...) to do an addition\n",
      "- np.random.randn(...) to initialize randomly\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: linear_function\n",
      "\n",
      "def linear_function():\n",
      "    \"\"\"\n",
      "    Implements a linear function: \n",
      "            Initializes W to be a random tensor of shape (4,3)\n",
      "            Initializes X to be a random tensor of shape (3,1)\n",
      "            Initializes b to be a random tensor of shape (4,1)\n",
      "    Returns: \n",
      "    result -- runs the session for Y = WX + b \n",
      "    \"\"\"\n",
      "    \n",
      "    np.random.seed(1)\n",
      "    \n",
      "    ### START CODE HERE ### (4 lines of code)\n",
      "    X = tf.constant(np.random.randn(3,1), name = \"X\")\n",
      "    W = tf.constant(np.random.randn(4,3), name = \"W\")\n",
      "    b = tf.constant(np.random.randn(4,1), name = \"b\")\n",
      "#     print(X)\n",
      "#     print(W)\n",
      "#     print(b)\n",
      "    Y = tf.add(tf.matmul(W,X), b)\n",
      "    ### END CODE HERE ### \n",
      "    \n",
      "    # Create the session using tf.Session() and run it with sess.run(...) on the variable you want to calculate\n",
      "    \n",
      "    ### START CODE HERE ###\n",
      "    sess = tf.Session()\n",
      "    result = sess.run(Y)\n",
      "    ### END CODE HERE ### \n",
      "    \n",
      "    # close the session \n",
      "    sess.close()\n",
      "\n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 42
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print( \"result = \" + str(linear_function()))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "result = [[-2.15657382]\n",
        " [ 2.95891446]\n",
        " [-1.08926781]\n",
        " [-0.84538042]]\n"
       ]
      }
     ],
     "prompt_number": 43
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "*** Expected Output ***: \n",
      "\n",
      "<table> \n",
      "<tr> \n",
      "<td>\n",
      "**result**\n",
      "</td>\n",
      "<td>\n",
      "[[-2.15657382]\n",
      " [ 2.95891446]\n",
      " [-1.08926781]\n",
      " [-0.84538042]]\n",
      "</td>\n",
      "</tr> \n",
      "\n",
      "</table> "
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 1.2 - Computing the sigmoid \n",
      "Great! \u4f60\u521a\u521a\u5b9e\u73b0\u4e86\u4e00\u4e2a\u7ebf\u6027\u51fd\u6570. Tensorflow \u63d0\u4f9b\u4e86\u5f88\u591a\u7c7b\u4f3c `tf.sigmoid` \u548c `tf.softmax`\u8fd9\u6837\u7684\u5e38\u7528\u7684\u795e\u7ecf\u7f51\u7edc\u529f\u80fd. For this exercise lets compute the sigmoid function of an input. \n",
      "\n",
      "\u4f60\u5c06\u4f7f\u7528\u5360\u4f4d\u7b26\u53d8\u91cf`x`\u6765\u505a\u8fd9\u4e2a\u7ec3\u4e60. When running the session, you should use the feed dictionary to pass in the input `z`. In this exercise, you will have to (i) create a placeholder `x`, (ii) define the operations needed to compute the sigmoid using `tf.sigmoid`, and then (iii) run the session. \n",
      "\n",
      "** Exercise **: \u5b9e\u73b0\u4e0b\u9762\u7684 sigmoid \u51fd\u6570. You should use the following: \n",
      "\n",
      "- `tf.placeholder(tf.float32, name = \"...\")`\n",
      "- `tf.sigmoid(...)`\n",
      "- `sess.run(..., feed_dict = {x: z})`\n",
      "\n",
      "\u8bf7\u6ce8\u610f\uff0c\u6709\u4e24\u79cd\u5178\u578b\u7684\u65b9\u6cd5\u6765\u53bb\u521b\u5efa\u548c\u4f7f\u7528tensor\u6d41\u4e2d\u7684\u4f1a\u8bdd\uff1a\n",
      "\n",
      "**Method 1:**\n",
      "```python\n",
      "sess = tf.Session()\n",
      "# Run the variables initialization (if needed), run the operations\n",
      "result = sess.run(..., feed_dict = {...})\n",
      "sess.close() # Close the session\n",
      "```\n",
      "**Method 2:**\n",
      "```python\n",
      "with tf.Session() as sess: \n",
      "    # run the variables initialization (if needed), run the operations\n",
      "    result = sess.run(..., feed_dict = {...})\n",
      "    # This takes care of closing the session for you :)\n",
      "```\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: sigmoid\n",
      "\n",
      "def sigmoid(z):\n",
      "    \"\"\"\n",
      "    Computes the sigmoid of z\n",
      "    \n",
      "    Arguments:\n",
      "    z -- input value, scalar or vector\n",
      "    \n",
      "    Returns: \n",
      "    results -- the sigmoid of z\n",
      "    \"\"\"\n",
      "    \n",
      "    ### START CODE HERE ### ( approx. 4 lines of code)\n",
      "    # Create a placeholder for x. Name it 'x'.\n",
      "    x = tf.placeholder(tf.float32, name=\"x\")\n",
      "\n",
      "    # compute sigmoid(x)\n",
      "    sigmoid = tf.sigmoid(x)\n",
      "\n",
      "    # Create a session, and run it. Please use the method 2 explained above. \n",
      "    # You should use a feed_dict to pass z's value to x. \n",
      "    with tf.Session() as sess:\n",
      "        # Run session and call the output \"result\"\n",
      "        result = sess.run(sigmoid, feed_dict={x:z})\n",
      "    \n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (\"sigmoid(0) = \" + str(sigmoid(0)))\n",
      "print (\"sigmoid(12) = \" + str(sigmoid(12)))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "sigmoid(0) = 0.5\n",
        "sigmoid(12) = 0.999994\n"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "*** Expected Output ***: \n",
      "\n",
      "<table> \n",
      "<tr> \n",
      "<td>\n",
      "**sigmoid(0)**\n",
      "</td>\n",
      "<td>\n",
      "0.5\n",
      "</td>\n",
      "</tr>\n",
      "<tr> \n",
      "<td>\n",
      "**sigmoid(12)**\n",
      "</td>\n",
      "<td>\n",
      "0.999994\n",
      "</td>\n",
      "</tr> \n",
      "\n",
      "</table> "
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "<font color='blue'>\n",
      "**\u603b\u7ed3\u4e00\u4e0b\uff0c\u4f60\u5e94\u8be5\u77e5\u9053\u600e\u6837\u53bb\uff1a**\n",
      "1. \u521b\u5efa\u5360\u4f4d\u7b26\n",
      "2. \u6307\u5b9a\u4e0e\u60a8\u8981\u8ba1\u7b97\u7684\u64cd\u4f5c\u5bf9\u5e94\u7684\u8ba1\u7b97\u56fe\n",
      "3. \u521b\u5efa\u4f1a\u8bdd\n",
      "4. \u8fd0\u884c\u4f1a\u8bdd\uff0c\u5fc5\u8981\u65f6\u4f7f\u7528Feed\u5b57\u5178\u6765\u6307\u5b9a\u5360\u4f4d\u7b26\u53d8\u91cf\u7684\u503c. "
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 1.3 -  Computing the Cost\n",
      "\n",
      "\u60a8\u4e5f\u53ef\u4ee5\u4f7f\u7528\u5185\u7f6e\u51fd\u6570\u6765\u8ba1\u7b97\u60a8\u7684\u795e\u7ecf\u7f51\u7edc\u7684\u635f\u5931. So instead of needing to write code to compute this as a function of $a^{[2](i)}$ and $y^{(i)}$ for i=1...m: \n",
      "$$ J = - \\frac{1}{m}  \\sum_{i = 1}^m  \\large ( \\small y^{(i)} \\log a^{ [2] (i)} + (1-y^{(i)})\\log (1-a^{ [2] (i)} )\\large )\\small\\tag{2}$$\n",
      "\n",
      "\u5728tensorflow\u4e2d\u4f60\u53ef\u4ee5\u7528\u4e00\u884c\u4ee3\u7801\u4e2d\u505a\u5230\u8fd9\u4e00\u70b9\uff01\n",
      "\n",
      "**Exercise**: \u5b9e\u73b0\u4ea4\u53c9\u71b5\u635f\u5931. The function you will use is: \n",
      "\n",
      "\n",
      "- `tf.nn.sigmoid_cross_entropy_with_logits(logits = ...,  labels = ...)`\n",
      "\n",
      "Your code should input `z`, compute the sigmoid (to get `a`) and then compute the cross entropy cost $J$. All this can be done using one call to `tf.nn.sigmoid_cross_entropy_with_logits`, which computes\n",
      "\n",
      "$$- \\frac{1}{m}  \\sum_{i = 1}^m  \\large ( \\small y^{(i)} \\log \\sigma(z^{[2](i)}) + (1-y^{(i)})\\log (1-\\sigma(z^{[2](i)})\\large )\\small\\tag{2}$$\n",
      "\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: cost\n",
      "\n",
      "def cost(logits, labels):\n",
      "    \"\"\"\n",
      "\u00a0\u00a0\u00a0\u00a0Computes the cost using the sigmoid cross entropy\n",
      "\u00a0\u00a0\u00a0\u00a0\n",
      "\u00a0\u00a0\u00a0\u00a0Arguments:\n",
      "\u00a0\u00a0\u00a0\u00a0logits -- vector containing z, output of the last linear unit (before the final sigmoid activation)\n",
      "\u00a0\u00a0\u00a0\u00a0labels -- vector of labels y (1 or 0) \n",
      "    \n",
      "    Note: What we've been calling \"z\" and \"y\" in this class are respectively called \"logits\" and \"labels\" \n",
      "    in the TensorFlow documentation. So logits will feed into z, and labels into y. \n",
      "\u00a0\u00a0\u00a0\u00a0\n",
      "\u00a0\u00a0\u00a0\u00a0Returns:\n",
      "\u00a0\u00a0\u00a0\u00a0cost -- runs the session of the cost (formula (2))\n",
      "    \"\"\"\n",
      "    \n",
      "    ### START CODE HERE ### \n",
      "    \n",
      "    # Create the placeholders for \"logits\" (z) and \"labels\" (y) (approx. 2 lines)\n",
      "    z = tf.placeholder(tf.float32, name=\"logits\")\n",
      "    y = tf.placeholder(tf.float32, name=\"labels\")\n",
      "    \n",
      "    # Use the loss function (approx. 1 line)\n",
      "    cost = tf.nn.sigmoid_cross_entropy_with_logits(logits=z, labels=y)\n",
      "    \n",
      "    # Create a session (approx. 1 line). See method 1 above.\n",
      "    sess = tf.Session()\n",
      "    \n",
      "    # Run the session (approx. 1 line).\n",
      "    cost = sess.run(cost, feed_dict={z:logits, y:labels})\n",
      "    \n",
      "    \n",
      "    # Close the session (approx. 1 line). See method 1 above.\n",
      "    sess.close()\n",
      "    \n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return cost"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "logits = sigmoid(np.array([0.2,0.4,0.7,0.9]))\n",
      "cost = cost(logits, np.array([0,0,1,1]))\n",
      "print(logits)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[ 0.54983395  0.59868765  0.66818774  0.71094948]\n"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "** Expected Output** : \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **cost**\n",
      "        </td>\n",
      "        <td>\n",
      "        [ 1.00538719  1.03664088  0.41385433  0.39956614]\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 1.4 - Using One Hot encodings\n",
      "\n",
      "Many times in deep learning you will have a y vector with numbers ranging from 0 to C-1, where C is the number of classes. If C is for example 4, then you might have the following y vector which you will need to convert as follows:\n",
      "\n",
      "\n",
      "<img src=\"images/onehot.png\" style=\"width:600px;height:150px;\">\n",
      "\n",
      "This is called a \"one hot\" encoding, because in the converted representation exactly one element of each column is \"hot\" (meaning set to 1). To do this conversion in numpy, you might have to write a few lines of code. In tensorflow, you can use one line of code: \n",
      "\n",
      "- tf.one_hot(labels, depth, axis) \n",
      "\n",
      "**Exercise:** Implement the function below to take one vector of labels and the total number of classes $C$, and return the one hot encoding. Use `tf.one_hot()` to do this. "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: one_hot_matrix\n",
      "\n",
      "def one_hot_matrix(labels, C):\n",
      "    \"\"\"\n",
      "    Creates a matrix where the i-th row corresponds to the ith class number and the jth column\n",
      "                     corresponds to the jth training example. So if example j had a label i. Then entry (i,j) \n",
      "                     will be 1. \n",
      "                     \n",
      "    Arguments:\n",
      "    labels -- vector containing the labels \n",
      "    C -- number of classes, the depth of the one hot dimension\n",
      "    \n",
      "    Returns: \n",
      "    one_hot -- one hot matrix\n",
      "    \"\"\"\n",
      "    \n",
      "    ### START CODE HERE ###\n",
      "    \n",
      "    # Create a tf.constant equal to C (depth), name it 'C'. (approx. 1 line)\n",
      "    C = tf.constant(C)\n",
      "    \n",
      "    # Use tf.one_hot, be careful with the axis (approx. 1 line)\n",
      "    one_hot_matrix = tf.one_hot(labels, C, axis=0)\n",
      "    \n",
      "    # Create the session (approx. 1 line)\n",
      "    sess = tf.Session()\n",
      "    \n",
      "    # Run the session (approx. 1 line)\n",
      "    one_hot = sess.run(one_hot_matrix)\n",
      "    \n",
      "    # Close the session (approx. 1 line). See method 1 above.\n",
      "    sess.close()\n",
      "    \n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return one_hot"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 48
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "labels = np.array([1,2,3,0,2,1])\n",
      "one_hot = one_hot_matrix(labels, C = 4)\n",
      "print (\"one_hot = \" + str(one_hot))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "one_hot = [[ 0.  0.  0.  1.  0.  0.]\n",
        " [ 1.  0.  0.  0.  0.  1.]\n",
        " [ 0.  1.  0.  0.  1.  0.]\n",
        " [ 0.  0.  1.  0.  0.  0.]]\n"
       ]
      }
     ],
     "prompt_number": 49
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**: \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **one_hot**\n",
      "        </td>\n",
      "        <td>\n",
      "        [[ 0.  0.  0.  1.  0.  0.]\n",
      " [ 1.  0.  0.  0.  0.  1.]\n",
      " [ 0.  1.  0.  0.  1.  0.]\n",
      " [ 0.  0.  1.  0.  0.  0.]]\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>\n"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 1.5 - Initialize with zeros and ones\n",
      "\n",
      "\u73b0\u5728\u60a8\u5c06\u5b66\u4e60\u5982\u4f55\u7528 0 \u548c 1 \u53bb\u521d\u59cb\u5316\u4e00\u4e2a\u5411\u91cf. The function you will be calling is `tf.ones()`. To initialize with zeros you could use tf.zeros() instead. These functions take in a shape and return an array of dimension shape full of zeros and ones respectively. \n",
      "\n",
      "**Exercise:** Implement the function below to take in a shape and to return an array (of the shape's dimension of ones). \n",
      "\n",
      " - tf.ones(shape)\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: ones\n",
      "\n",
      "def ones(shape):\n",
      "    \"\"\"\n",
      "    Creates an array of ones of dimension shape\n",
      "    \n",
      "    Arguments:\n",
      "    shape -- shape of the array you want to create\n",
      "        \n",
      "    Returns: \n",
      "    ones -- array containing only ones\n",
      "    \"\"\"\n",
      "    \n",
      "    ### START CODE HERE ###\n",
      "    \n",
      "    # Create \"ones\" tensor using tf.ones(...). (approx. 1 line)\n",
      "    ones = tf.ones(shape)\n",
      "    \n",
      "    # Create the session (approx. 1 line)\n",
      "    sess = tf.Session()\n",
      "    \n",
      "    # Run the session to compute 'ones' (approx. 1 line)\n",
      "    ones = sess.run(ones)\n",
      "    \n",
      "    # Close the session (approx. 1 line). See method 1 above.\n",
      "    sess.close()\n",
      "    \n",
      "    ### END CODE HERE ###\n",
      "    return ones"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 50
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print (\"ones = \" + str(ones([3])))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "ones = [ 1.  1.  1.]\n"
       ]
      }
     ],
     "prompt_number": 51
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output:**\n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **ones**\n",
      "        </td>\n",
      "        <td>\n",
      "        [ 1.  1.  1.]\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "# 2 - \u7528tensorflow\u6784\u5efa\u7b2c\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc\n",
      "\n",
      "\u5728\u8fd9\u4e2a\u4efb\u52a1\u4e2d\uff0c\u60a8\u5c06\u4f7f\u7528tensorflow\u5efa\u7acb\u4e00\u4e2a\u795e\u7ecf\u7f51\u7edc. \u901a\u5e38\u6709\u5982\u4e0b\u4e24\u6b65\uff1a\n",
      "\n",
      "- Create the computation graph\n",
      "- Run the graph\n",
      "\n",
      "### 2.0 - Problem statement: SIGNS Dataset\n",
      "\n",
      "\u67d0\u5929\u4e0b\u5348\uff0c\u4f60\u548c\u4e00\u7fa4\u670b\u53cb\u4e00\u8d77\u51b3\u5b9a\u53bb\u6559\u7535\u8111\u7834\u89e3\u624b\u8bed\u3002\u4f60\u4eec\u82b1\u4e86\u51e0\u4e2a\u5c0f\u65f6\u5728\u767d\u5899\u524d\u62cd\u6444\u7167\u7247\uff0c\u7136\u540e\u5f97\u5230\u4e86\u4e0b\u9762\u7684\u6570\u636e\u96c6\u3002\u73b0\u5728\u4f60\u7684\u5de5\u4f5c\u662f\u5efa\u7acb\u4e00\u4e2a\u7b97\u6cd5\uff0c\u4ee5\u4fc3\u8fdb\u8bed\u8a00\u969c\u788d\u4eba\u58eb\u548c\u4e0d\u61c2\u624b\u8bed\u7684\u4eba\u8fdb\u884c\u6c9f\u901a\u3002\n",
      "\n",
      "- **\u8bad\u7ec3\u96c6**: 1080\u4e2a\u56fe\u7247\uff0864\u00d764\u50cf\u7d20\uff09\uff0c\u4ee3\u8868\u4ece0\u52305\u7684\u6570\u5b57\uff08\u6bcf\u4e2a\u6570\u5b57180\u5f20\u56fe\u7247\uff09.\n",
      "- **\u6d4b\u8bd5\u96c6**: 120\u5f20\u56fe\u7247\uff0864\u00d764\u50cf\u7d20\uff09\uff0c\u4ee3\u8868\u4ece0\u52305\u7684\u6570\u5b57\uff08\u6bcf\u5f20\u657020\u5f20\uff09.\n",
      "\n",
      "\u8bf7\u6ce8\u610f\uff0c\u8fd9\u662fSIGNS\u6570\u636e\u96c6\u7684\u4e00\u4e2a\u5b50\u96c6\u3002\u5b8c\u6574\u7684\u6570\u636e\u96c6\u5305\u542b\u66f4\u591a\u7684signs\u3002\n",
      "\n",
      "\u4ee5\u4e0b\u662f\u6bcf\u4e2a\u6570\u5b57\u7684\u793a\u4f8b\uff0c\u4ee5\u53ca\u5982\u4f55\u7528\u6807\u7b7e\u8868\u793a\u793a\u4f8b\u3002 These are the original pictures, before we lowered the image resolutoion to 64 by 64 pixels.\n",
      "<img src=\"images/hands.png\" style=\"width:800px;height:350px;\"><caption><center> <u><font color='purple'> **Figure 1**</u><font color='purple'>: SIGNS dataset <br> <font color='black'> </center>\n",
      "\n",
      "\n",
      "\u8fd0\u884c\u4ee5\u4e0b\u4ee3\u7801\u4ee5\u52a0\u8f7d\u6570\u636e\u96c6"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Loading the dataset\n",
      "X_train_orig, Y_train_orig, X_test_orig, Y_test_orig, classes = load_dataset()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u66f4\u6539\u4e0b\u9762\u7684\u7d22\u5f15\u5e76\u8fd0\u884c\u5355\u5143\u683c\u53bb\u53ef\u89c6\u5316\u6570\u636e\u96c6\u4e2d\u7684\u4e00\u4e9b\u6837\u4f8b\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Example of a picture\n",
      "index = 0\n",
      "plt.imshow(X_train_orig[index])\n",
      "print (\"y = \" + str(np.squeeze(Y_train_orig[:, index])))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "y = 5\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "display_data",
       "png": "iVBORw0KGgoAAAANSUhEUgAAAP4AAAD8CAYAAABXXhlaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJztfWuMXdd13rfucx6cIWdIiiJFSqJM\nPSzraTOyXKuOYtmC/IiFooYRJyjUQoD+uIWDpojtFiiSogXsP3H8ozAg1G70w40fSRwJhpFYUWQ0\nbmzZlPWwHqZJSZRIiuSQnBnO477v2f1x79y91jr37Dl3HnconPUB5Oxz9z777LPv2festdda3yLn\nHAwGQ7aQ2+oBGAyG4cMWvsGQQdjCNxgyCFv4BkMGYQvfYMggbOEbDBmELXyDIYNY18InogeI6CgR\nHSeiL27UoAwGw+aC1urAQ0R5AL8B8FEApwD8AsBnnXOvbNzwDAbDZqCwjnPvAnDcOfc6ABDRtwE8\nCCBx4U9PT7n9V121jksCRKlbrus6G9dl+pN4y9DPcahd+unZhPm57LFGL9VNdm7l3ad9BpJqT548\njdnZ2VW/3PUs/KsAnGTHpwC8P3TC/quuwg//9nudA/3gpXyaKfjAUqp2REzDiQ2DkutYny65mfwi\nY+OghLKcEjkFoXYUaJkMOa7k74ISPo/1Fzh2oUEFnm5RlVIyjbVywdp0faaVilO304dRrxxe+Oz5\nc5Gocd3WD3zswVRD2PTNPSJ6hIiOENGR2dnZzb6cwWBIgfW88U8DOMCO93c/E3DOPQrgUQC47dZb\n9AuqB2I/by7wyuc/qqG3erwH1gerJfXbJ8UuSqzjB/E3Wtr3rhylS3o1qo83XvIMzDf/dAC9gjcV\n39kAo0gLFzjaKgSnKjYJ/hlc857bgO3X88b/BYDrieggEZUA/B6AJ9bRn8FgGBLW/MZ3zrWI6N8D\n+HsAeQDfdM69vGEjMxgMm4b1iPpwzv0QwA83aCwGg2FIWNfCXx+0Zpa8yyz1btf3c30eBTQ/Cu6d\ncv1W7pyS2MpP3IIXH8R6T7svQfxeQg1VVbptgrC5MEEpj9/mANv8Ka4bRv89mninae80eX8lbqFI\nZ2SLEmvC951Up/eYkg1//fbFwjCXXYMhg7CFbzBkEFso6gdElaA4nCx7pnVGE9cawO7i0F8cDF9W\n9eHS3ae8brjPNDXx2wyJjf0P4sMNqF1B8bt/w0FE29V7i58V9uUJqZBpr5esEoTuhBIex5D4vl6j\npb3xDYYMwha+wZBB2MI3GDKIoev4KypuSL3VbovSBLaGvQAEdF9K1y6EtJF08RPXuEkR0hiTbUOB\nS4dCgkJdc/Om7mPwmQyb6VxSBUKznBToE/cdd33bdboPmVbTatvs2muL5QnWDqrz2xvfYMggbOEb\nDBnEFpjzHPufIznAPdnrLuDhp6VG0X060TAejcbFNR77n9hdXKwLxPEniYOx8Qaj4hKEvoAsqL0J\nU2oLwdpkcolklSA29qS5cqHxao+8/ubC+L2ke/5CYrpTd5rUv4s9nP0vHTaCrk9FtTe+wZBB2MI3\nGDKILRD1UwglOvgjyZsuIEaHugh5cMnYm/XvVMdP6U9yASjxPhQcI3pLu587CGnJ4IiRliROeLLH\nZnyfPaAGJPUR0xYSSEUSe1tlx3/VtimwRre7sCoxGOyNbzBkELbwDYYMwha+wZBBbFl0XlBDSUsu\nORAPxFo01+RxCG0rdUiY+mBNnnrQtsm19RE0bSUheeMk5HU3uG9b+MwY/wf7IGQqS/Hxqo3XGv2X\nfg7SfhuDUKvEYW98gyGDsIVvMGQQww/SWRFJAva2kOgsTTeDeLSxZoPzO8Q64SLlYGa+dHY6FyCj\np+BcpRXhkwNWko5i5s2UKsd6SSNW6T6I9HQYyWelbimCgMJ9rmlYQZD6G4a98Q2GDMIWvsGQQdjC\nNxgyiOGb81b0mZjJbg0mqvQqPpJ02jj/frKOnGiuGcSyEgv5S2gWcrENEXZyPTPllMYjGROMVIFo\nxXBK3PXDrXnCEwgwUpr9Ngyp7Xnp7jNuPnWxNiGs+sYnom8S0QwRvcQ+myaiJ4noWPfvVKqrGQyG\nywJpRP2/APCA+uyLAJ5yzl0P4KnuscFgeIdgVVHfOfd/ieha9fGDAO7tlh8D8GMAXxjs0iFRVrVM\nSQKX3qEtmf9MCrnpdIkg2UayJN7nvpJuICTart8jL+R1lzbl12YI+smq1QaI6cFzNJnHGrof4KTk\nxztkVkyOckyDtW7u7XHOnemWzwLYs8Z+DAbDFmDdu/qu89OW7E5P9AgRHSGiI7Ozs+u9nMFg2ACs\ndVf/HBHtdc6dIaK9AGaSGjrnHgXwKADcdustPQEoPYEEBL9diJRM8rIhUBm4VkhUXoNsGxL5XMzz\nMGGXPGAJSM0fmNhqAIRUk5QYQMIOtAtTkyQfp9v9jz9YKb3u1oi1BDElB6FtrufeEwAe6pYfAvD4\nGvsxGAxbgDTmvL8E8FMANxLRKSJ6GMCXAXyUiI4B+Ej32GAwvEOQZlf/swlV923wWAwGw5CwhUQc\nybpYyHiVVo8PR5Kl04PSOuBp8ocQ9z9S1agLrFVDF12kJ7kM2JfStQueFmJZiWRV4hRow1ayR1v6\nyMmUewHBiMqUl0rteRjy3FsfzFffYMggbOEbDBnEkEV91xPFKGTmCsqeKT3+Qra4kItf2j4Sx6e9\n85Ll42BcS0CW4950sczCCd5/QU+ymHshLya7Ggp1J9Zniv50sziZXp/BxpHo4QcgYupDyPNtrUgm\n+lBI+10E+guquQPC3vgGQwZhC99gyCBs4RsMGcRQdXwHpt8E3SDXF3kExM04LkFBipt70pFLBCxl\nojbslquqUoYhBnk4gvfWv8v47DLTamC8IXIMqbunGka4LqVenOzKqnTrDSPiSHliaCMiOdIl0CyB\niCPlPNkb32DIIGzhGwwZxHDNeY6LWyE3sGR2DCnmDmCSSTDhBUk0Urqqxcwzac10QetVyOMv4MGV\nmF9b27m8mcs56TGXyxd9XUqe/jiZx1rMZWmj7gKm4NgHrn/tJnAEhiMPB4koXPu102os9sY3GDII\nW/gGQwZx2dBr92uSuq+VLtcUNKL6CLbrv50ek9gDIrZkrg4EzgiJPb0oKDfh/UG7XhXtFl/6Wa/c\nujQn6sZuuL1XHj9wiHWejoNw5eppEN6ETqpMvwWf5Ps2kMNm6v7T1aRvO4gqO8DlYG98gyGTsIVv\nMGQQtvANhgxiy4g4QoT2cf1rA5ghRP8ba8rRowh5z4UiCFN7uwkezuROolazV557+eeiWe1YLzES\nmpWKqJs977lTr/tdz5xe2rZdDaT/dWPjTa5aI9bv2RkyP4Z724i7STf+ICFt3G450AjsjW8wZBC2\n8A2GDGILRH3H/vfgxBxp0zatVbwM0Letgv7GvrUGocT5RtIRjoSuELVavfLsr5/vlZdeOyratZv+\nvKXlmqirLC73ynsX5nvl4rbJxFFolSO9mSudm2Nq6oqUxCRhPSvlxWJNg5FPA19gbabOdLA3vsGQ\nQdjCNxgyCFv4BkMGMXQdv8fDESSyTEmLGDNlpSSeCAUCBrtIq8MlXLhzheSWAYKNpC7bzGQHABde\nfbZXXjr6gu+vJSPwlpe9C+9yoyHqKpzMM5fve93YkELEKiGoyEABSviiAspvfNr678VsmEF3jcSZ\n6ZB+lIP2niaF1gEiepqIXiGil4no893Pp4noSSI61v07NeC1DQbDFiGNqN8C8EfOuZsB3A3gc0R0\nM4AvAnjKOXc9gKe6xwaD4R2ANLnzzgA40y0vEtGrAK4C8CCAe7vNHgPwYwBfWLW/rlAyEKV8ogwc\nYiELqAsBmS8UCZeeD513P0hEGx9H8rXajXqvfPbFfxZ1y6+/3CuX2Ndbq0txvlrzon61IdWF/O69\nvfLI5A4/poFE7HS1YR65hIMgZV06c97aLWVrU2lCEZvJUYMBo+g6HQgH2twjomsB3AngGQB7uj8K\nAHAWwJ6E0wwGw2WG1AufiLYB+GsAf+icW+B1rvMq6PsbRESPENERIjoyOzfXr4nBYBgyUi18Iiqi\ns+i/5Zz7m+7H54hob7d+L4CZfuc65x51zh12zh2enrL9P4PhcsCqOj51/Ge/AeBV59yfsaonADwE\n4Mvdv4+nuWBPhxkkcZygo0ln8wqkrAu7+qZUnkIqmxjuABF4SdsXjeqSaPf2c//UKy8c+5WoGy+N\n+PMir7tXlpdFuzrT65caLVF33fW39MoF1l9obtJHtA2SHy9Jrx/ARBocV3/EouJSW3FT3kugzoXy\nOgroqM/B7jSNHf+DAP4NgF8R0Yrz939GZ8F/l4geBvAmgM8MdGWDwbBlSLOr/xMk//bct7HDMRgM\nw8Dwo/NW5Nk1u04FZPYk7zxdmTbV9hohrqQ6lCQayYao+vJir3ziZz8S7ZbePNYrF9t5UVdte7G9\nxjzylhYWRbvluo/Iiybk3suB99zRd0xh8odACq1AroI1RaDFQzuTO0wiYFljRGX61snfbYicVaZV\nT36+B4tyjMN89Q2GDMIWvsGQQQxd1E/c1d/4jEbJYLJVmLM+tAOdTlYMeSFq77z6knePOPmLf+yV\nl04eE+2aVS/CL1fljny96Y/r3DuvKnn1m3n/1d/xkd8VdeM7ptlwA/eZ1kQRmI+1eEPGkNajMC1j\nijYJsXRjUUOSlvBjF7V75Xx5TI5jZLTvpfQow96RA/m7BmFvfIMhg7CFbzBkELbwDYYMYgtz563P\nHJH2MskfJFVwfvWYRpqyj3SkEbVLF8TxzAs/6ZXb5071ykUnf5+b5I8vKU78+TlPjtlusYg8lvoa\nAK656/298rvufD+S4AK6b4guJVFX1U5xwrwZapzMex/WfLlXHCsrApPGpYu9cv3cW6KudeGcP6hI\ns6hjJtMcvwGm0wPA2K1+jstXXhsb5QoCwafqeQzka0gBe+MbDBmELXyDIYPYshRaIdNEagexoAkp\nFAEjR5J6IAkeYnGxltVF0ty2PONF+IWjR0RdbsmL6cTE+WpN9nFp0Yv3NWVeWmJmuya7t5vf/wHR\n7rc++a975ZISS9MiSC6RkCssJKanDoDRl2Jz3K7JYKTmAhPhL5zplRsX3hbtlmd8Xb4mSUvKOb9M\ncpCekpwx0OX8wOoz50W7xYY39e2/f5+oyxW9GhYW2UOzZeY8g8GwCmzhGwwZhC18gyGD2LrceTFl\nJkCimVSl9fYASUdqukcxLlUXJZiGXFu0ay5719vFE6+Iuvq5E71yrilNStW672dpibnb1uuiXZtF\n4LXVNBYnJnrl2z50f6/8vo98XLQrj0qXUg6punstNq5/ukBdOvA8ibGciazTxsJsr1w584Zo1jj3\nZq/cWlT0bnU/j9T285sj+c7LtXxdpSLnuxF5nT9S99lgH7TYc1CtSxfp8TFPWrpP5RJICsiLpUAP\nwMx5BoNhVdjCNxgyiOGn0FqRcgJEGbGIuSQe/Ji2MHiu47iIxEXbZBNVu+XFwcrbr4tmyyc8t32h\nKc1Lo2zKl5pSRVhgaa2qNd9/sy3NeU0mUubGpMj+wU892Cu/6867/DgK0nPPsYizeMRc1LddpFQa\nPj9xaykX4f37JZeT75qIidiNeenJuPCGV5NqZ714X52fF+0KLNKwmJfmNvEdsmvp8TaYB96C4ieM\nmnwO5HmXeCoypkpsu3KvaHfd4Xt65XyxhEQIq6X2lNw471Z74xsMGYQtfIMhgxiuqO8coq4YGXes\n49uZgURCgXbypJSkBSGvO1XDSRfmj/2yV66flkQZYlILUqyrMrFxqSp39ZtM9G+yHf9aTXrnNSJ/\nb3d87F+JumtufZ8/YFPQVEEpYrcecpdZWixYndqNlpvR8h3Cv0/HUn41Zs+KdtVTr/XK0SXp7VYE\nE50Lvv9WWXoaLiww+vGWVEfyOT+ONptTrbbUGQfhxUtS1K/UfNviyLiom9x/ba/8njv83B949y2i\n3fjkdn/gkp/vjaPaCMPe+AZDBmEL32DIIGzhGwwZxFB1fAcg6po8XIiIQ+vugko/ZS6sEHm50OMD\nUU6R1APnXnuxV66f8bppXlsfiaWnbki9eJF55C0tytRYdRZZV2d6veoCN9/3iV55/823i7o2i1Rz\n7WT9PEwcyvTzNtOLa9IbzbFIOF1Xm/P6emvWE1nk6lJ/LjEzY1GZuQi+rs68FyO151Gv+mjF5WU5\nDo5my89NRXlD0qjX3XfcIOf0uutu7JX3XH1Q1E3tubJXLpTKidfmiM92f1P2IMY7p/6uhlXf+EQ0\nQkQ/J6IXiOhlIvrT7ucHiegZIjpORN8hooBx0mAwXE5II+rXAXzYOXc7gDsAPEBEdwP4CoCvOucO\nAZgD8PDmDdNgMGwk0uTOcwBWZNJi958D8GEAv9/9/DEAfwLg66t05gNMSJt//LELiPpCFIqZ89Ly\n3nMRWAcE+brl82dE3cJbv+mVczxVVVuK0XnmPRYpCbtR9x5idRXI0WDHNSY6H7znftGOi/etphR7\nhdcdE9NdS5JLcM45p7niWZBR6wIzv6kAmCI3A+bkd0FsTnJsEsoj0tOQe/I1m9JDsbrE58OrCDUl\npi8x0f/CsuQgbDPijMk9ngDj0K3vFe0OvPvWXnn7rj1yjMobkCPRmy72XCU/jwmcJUHEVNQBZf1U\nm3tElO9myp0B8CSA1wDMO+dWvqlTAK5Kd0mDwbDVSLXwnXNt59wdAPYDuAvATWkvQESPENERIjoy\nOze/+gkGg2HTMZA5zzk3D+BpAB8AsIOot329H8DphHMedc4dds4dnp7a0a+JwWAYMlbV8YloN4Cm\nc26eiEYBfBSdjb2nAXwawLcBPATg8dX6cnA+vxhJ5TcSrrj696i/qS9kioulY046UrpYs+pNbHPH\nXhB1LWY2ajSY2UwpZiMjXicsKJfdfMG3HR+VdTsnvP57acnzt9cuSJ73kz//B9+fMtPl2l6XbzM3\nVCgdv8hcYDlBBQBMlLwZbYT8vRTV91IsePNVriyj/xotP65FFmlYqUvX4Yhdu6l09xrLGVDn0XNV\n2a5a8Ka4fe+Tuvuh2w/3yjv3em00RjBKazWjsWcupF+70HPLhsHJXgYYSKjPfkhjx98L4DEiyqMj\nIXzXOfcDInoFwLeJ6L8DeA7ANwa6ssFg2DKk2dV/EcCdfT5/HR1932AwvMMw5Og8IOqaeUiZfxw/\nDhCIiZRLMb48dTF98X79Ke+82eMv9cqNS7OirtXsz9mm+ds4cUNbieKjjOuurGTDMnMBrNQu+c8r\n50S71lveK057/5WZmF5iUWxFxbGXL3huvpbiii+OTvbKk6O+j4JK5ZVn5rx2JEX4tmPHLa9yaD67\nhQWmPjWlOlJn0XSlnZ7YYt+d8j20++ANfrzTu0SdIP4Qj1wy2YtW3UK+osKAzPqIlGrF1cS2SsPV\nXmCb3su+nC9K9am013sNFqalyXFQjg7z1TcYMghb+AZDBjFkzj0Hz2mXi1etIMDHBx1rIk4LBd/0\nx+JZuWNeOeupmp3yJGuIHWm2K05SVG4zr7h8XqVjKvqbKygyiOKIF+32TE/7c8rKc4zdWnNiUlRV\nKixwhnFvF8sjol2eqQH1uhzjUsOfN8YsFHHVitFfq8AZvkPPrQs8EAmQKb9Ku6UP2IGbvIfiFe/y\nriOj27aLdoKiW6uQnEOReRPGPO6YShYp0pJ2w4+xVZVBRvV5n6Jracan5Wqx7LsAUGBqTFGpsgVG\nrJIXQTryYV94zXuOTn/oY6KuOCVVnNVgb3yDIYOwhW8wZBC28A2GDGLovPpeHUs2mWg+e6GvB3jH\nRWUswM9/UFvyprKLx34l2vGou7aKuhPWN15WP5/S/CPHmGP6aLEkvcdyOa9PlwojrJ3qo8D07prc\nJ6C8P4+TcuQL8qsW5lNljmxw4smLvo+yuhduznPanMcJQdg0FhSv/q6Dh3rl/f/yE6KuPOZNjjzi\nUZOKNCrepLl07qSoWzrjcx5w4pCCirgrsnGRMsWB5VCgltz3EUNh3orUkN/LCNtTcU05jzU23wX2\nXbSdSrFe99GRhVNvirqdO3ZjENgb32DIIGzhGwwZxPBTaHWlHNIRDcIlLzl1VYj3XgTpqAiHiJFS\nzBxl3HkLklwiz9IxtZrKdsg917jKoVJhcW8xp9SFep2bwKRI2WCBMyUmmisHLiHq5/Lyt5tnz80x\nVaKtzFxVHgCjxtFu9Q+Kqqh74emqCgU5Dm5GixgHoaalK+3c2Struos64+BvzHtvxdoFGQhan2Oc\nfg1JxFFi89Ni4y+UtSejn2RSebJynJuf5JJps2ez3vDP2PzsJdFuNvLHefXgcrKWiD1YEeRzVWMN\np1Py+yXB3vgGQwZhC99gyCBs4RsMGcTQdfweYsnzAnUicirkiptMdjB70vPgL5z2Za3P1Vk+O6XS\ngph+l2N7CJqggljEX1uxbTbYuPLKJbPZ4NFuvg/N9VjiRBkjUtcbGfPmvCozPS1VpEvtwqKPECu0\n5RyMM4KNKM9ddhXYtaNmctyaIFlRewH1s97cdu7CKdkFc3PNs/kYU/fMIwjz268QdVT0bRcXvf5/\nQeng5Py1tKusiODMyS+jyfT6SsW79p4/LyM78+wdG6kHq8pIXSrsuxjfJd1wbzx8d6+845rr5RgD\nEa39YG98gyGDsIVvMGQQWyfqK7iQOM/qQlwKXLxcnr8gas686tNaRy2eZkr2wCPadOrniHn15Vi6\npyg23GTetAbjjtMX56m4uJlIp+hC5MVN15AeczV2bzwl93JNEmC0m/54VHHdF5gJjJuX9L1EnHhC\nqQt51keO8Qzq4ErOv1+AVEdGxjyXHteYtFQbsei2lubjW/DeejMz/pk4P6fIMBj3X0Gl8uLPo/6u\nG+y8GnO0qzq5tMa3eS/Ese3Tou66a6/rla9knozTLD0XAIyNb+uVKcZLaaK+wWBYBbbwDYYM4rIR\n9SU1dlJNmK+DE2CceeVZUcc99CLuIxZpjzOPnBJM81y8YrJnQ42EEzk4FVDCd+vbbSmmF5n3mEjb\nVJD9N5h4XNepsVi5ycTQtgoumWDZYcfGVFordj9clG0pYpIiG0e5KB+lUskfR8wTk9R8RIzEZFmp\nLct1H3zDA6acUpF44I/e3a4wkpG5RS/2zy4pQo2W77M8IsdY2sY4CHdJ8Xsvy567c9+BXnl8+5Ro\nN8I4D4tlaZXI59jcCadJHchGfdsBGIyLG/bGNxgyCVv4BkMGYQvfYMggtlDHT0jzi9VSY3HIo8WL\nPkpr5sRvRB034eVzXscq5OUURMxeoz33HNO780yXbKmG3BSndeuI+nvndY5ZtBuz4XEyxs5AfFHx\nWqDFdHKuC+9QeQvLJW+yyqs5aDCTWK3ux19QdkVix/miHEiLmT6J7VHoqEzHxru0LIk4q4zctMV0\ncP188H2UpoqUrDFCjCW2R1HPSfLRa27zabLfdcvtom7X3v298igzywFxgpOkMYatbWvKk73aB0Gk\nfuN3U2U/R0Q/6B4fJKJniOg4EX2HiEqr9WEwGC4PDCLqfx7Aq+z4KwC+6pw7BGAOwMMbOTCDwbB5\nSCXqE9F+AJ8A8D8A/Efq2Bk+DOD3u00eA/AnAL6+em8dkcTFzA/JGWwTLRXKNDR37kyvvLiwIOq4\nqSzPM+7mdSQO84pTblotfsxEPKcCcbiHn9OirdO+a+w8xrEWsZtuqdRVOSSbr7gpbpJx8xfyks2j\nXmNc8dAZbBmJRsQCZQqyD26OXK7JPhzz5Mtxz0tFCNJi6khNmQsvMpNb0/nvZUTx6o9O+OPJHTKw\nZd9Oz0W3jXHPT10hzXITU36ucjlNCcIQkMTTBsoE2wnTp65LvrhXLdKNIe0b/88B/DG8x+VOAPPO\n9Z7UUwCu6neiwWC4/LDqwieiTwKYcc49u1rbhPMfIaIjRHRkbv7S6icYDIZNRxpR/4MAPkVEHwcw\nAmASwNcA7CCiQvetvx/A6X4nO+ceBfAoALzn3TcOtvVoMBg2BasufOfclwB8CQCI6F4A/8k59wdE\n9D0AnwbwbQAPAXg81RVdf10kdJTYldLB5+e8W+7isoz0GmVeksUci7bKS/NPiUVmOaVbN4W7bbJ5\nibfTGj0nodT6bp7p9dysiJzS45kZTXWBiUmv7xYYIePSotzzaDNTWTEnH4MRNlk5ZqzJq3acZKRe\nka7DnB++xF2Rc3KfoMX5+HNytg7c+lu+fNsHeuXxHTK6rchMkznFWkIJ5CwxbZnv+ygdPKBaJz6p\ncWvb4CSx8b2tjXtvrseB5wvobPQdR0fn/8bGDMlgMGw2BnLgcc79GMCPu+XXAdy18UMyGAybjaGn\nyV7hLwsJLVrE4RYwLq5VlySZwlvHj/bKlxZU9NUoi75iIvBIXpuouCiupof6i2FtdTeNphdfI2W+\n4yQXBU2YL1zyeHpnlSaLjWNERdZx01+VRaPVq1IU57z9sfRaTEzPsTGOq2u169wbUkXnMW/AMuPI\naytXQ8fSR02oHAH7bri5V57ay41Gg0Wi9TsrZikLeo7ydiGTWvJR8Inn15YDSR7HGudgBearbzBk\nELbwDYYMYqiivnNAFPX3XIsCcpgg4mDnnzr6kmh3nmUQXVZ00o2W76VQ8KI4D1YBgPGREdZOk0v4\ntlz41sQQfMdfi/p5JurqX90i25DmRBYjozKghHuWxcga2PXKLNVWaUySP4yOeEpqzTHXijjBBvOY\nK8t23CKybVTx9nFzAys3lNoS8THqHf83fKqzxZq3SuTGJkU7nioMSnVzbK4KLGNtYURlKuaBSjE+\nu2QCDPGocmtAzDsvnQgvL5tenB90v9/e+AZDBmEL32DIIGzhGwwZxBYQcaxoI1o35S2UYYTpnOdP\nneiVf/Ps/5PtOHmlUo9qjHSRk0TGIgHZcUHp/wWmn+aZ3losSm8xnrq6oCK9xlj6q4ltUi8eZ6mg\nOKd8QRFZ8jHmC7L/Ajc5cqIPkjptPs/yAqjf/zwj8NzGxqTvxbFowpIy04kISLYj4iLF70+MzFN9\naa1lb66tM30/r+ab2LgaLbmnssTJ7pkePzoh9wlKjHxU7Bmo46bKHxCxZ4KnTiMVaSj6U89mjj3v\n5QlPmFLe/y7RrjDhCTx11KcZseRnAAAU7UlEQVTx6hsMhlVhC99gyCCGKuoTuPkpZBeR4Omwfv3P\nT/XKi3MyI6kQ03M6wIYF5jDVQWcu5Rx5OgioXeDef2zqIh3o4+vGytIUt2vai3I7pyX3Ovemy7Hx\n62AervrklLcbN0txj798XovpvtxWv//cc2+UcelpTvw24w/UBCMtJhJzwo7asvS2rDOOvGpTqV1V\nrxZtG/WqyfiYVMG4AyRpczHLuNtmacS4mgIAUWXe96FUmjZ7DpaWKqKuyYKdtrFUZHnl4Udcu9Sv\nWxZ01WKqW/PsSdFs4q77/CnjUlVZuVxagd/e+AZDBmEL32DIIGzhGwwZxNDNeSvaTIhvsNWUkWRv\nPP9Mr7x48WzieZHOYczQZjooV+vzSifMs3Z5petxXvki0+MnlDvsFONenxiTOv425n5bVgQbRb4v\nwcw12ryZY3pgLpYXgI2fp6rWeisnhlBc9I6ThTLTXrMh9eJa1btF66nPkx8Xz2lYUVGCPF/ehQWp\nP0dsH2hqwpvbJhtyvktMydfmtoVF5rotoitlu1FmZi2V1L4JeyZ06nSee5FH7mlykya7z6aiZ3El\n/93wvAuVM6dEu/zM273ytoNSxx/UZ9fe+AZDBmEL32DIIIYu6rsV8S1GtuFllQun3xJ1MyeO+XZM\nXCupaLExVjd3aUnUcVMf92jTxBB5xjE3PiJFym3M026SifA7mBgKAOWSrxvTY2Sifq6gzGg8JTUz\nK7aVaagN5jE3otQFpo7we25IaR7gkXBqHMRE0Rzjy6O6JhXx99ZsSl79GhNt66xca8lrzS55Ufzs\n7Lyoi5jda7HiTXETS1J9Ko8wnkSdVo31Xy4zlSBKJkjRJlJi87i8JKM+wdSk5XIywUurxb8L5dXH\nufTZpdtKPZvkap32OF2R9VOK/PbGNxgyCFv4BkMGsYXZciVqFc8Pd/xXMnfH0rIX2yOejkmJdWNM\n5KtWpajVbHoRLc/kqbySjfhZo0rkm2Ci4vZRXx5RaguxnXCnMsw2miwLbk4RWzARkGeb1Z51nAuw\nsixVGk4PzjMBa24JTkDSUqpE1GZZcJk4n5OaD0pcdK6rwBlGhNJs85Rfst3obs+ld9vhj4q6RsPP\nY2XBJ2NZXpAem3NsDupVaRlYXGbZfmtevB+tq++dWUp09mCekrhWlSpCfdk/t+XIeyXmdRAaE9ub\nLal31RtejeHq67vvvke0G9/js/aGeAHTwN74BkMGYQvfYMggbOEbDBnE8HX8ro7eVqSLR196rld+\n/VVJojlSZCmjOFklKZMM0313bN8m6lrMXY+nkt6mouemJr1pbuekJMqYZHr9GPMWK5d1tBgjeFTc\n+Y6bjRRJJ09d5RhxRkOZyrh3nivIaxM/ZvsEsWgxpj+TCheLBJmF1yU5ASggPSwj9QopsGjAQoHp\n1gU5ju1XHuiVr/0Xvy3HKDYmWLSfioZssflpNqRnID/mpBk5ZcbNsQhIUhsinNA0UteuV3268RrT\n91st+Z2Ju46RyfoPxpjX5+59+0W7vCKGkZ0kp1/vh1QLn4hOAFhEh1y25Zw7TETTAL4D4FoAJwB8\nxjk3l9SHwWC4fDCIqP87zrk7nHOHu8dfBPCUc+56AE91jw0GwzsA6xH1HwRwb7f8GDo59b4QOsHB\nIYo64udbJ14XdS/+9J965cqyTH+VG/fi+Cgz2WlOeR58U1Sc+FMTXmyPmr7hpOKsn2LHU6PSfjXG\nPPkEr57ivSuxFF3x9FT+2qURee0iVwuYOF9QwTxN5tVXKEmvwYgY/xwTQ9uKcCRinmS5SHmSMTG1\nzTzT2i1NWsI88qoqjwEL6ImYShN70zACjJpKiZZnxBbyu5bzkWdecvkxqVqNjkuVr39/8WNRl1gz\nEPV9IpIC1gZJk6U9FldD2je+A/AjInqWiB7pfrbHOXemWz4LYM9AVzYYDFuGtG/8e5xzp4noCgBP\nEtGveaVzzhHFaD8BAN0fikcA4Mo9u9c1WIPBsDFI9cZ3zp3u/p0B8H100mOfI6K9AND9O5Nw7qPO\nucPOucNTO7ZvzKgNBsO6sOobn4jGAeScc4vd8v0A/huAJwA8BODL3b+Pr9ZXq9nE+XMd7eCXP3la\n1C3O+t+NvCKXFDpt0/9WtZRJps3MV5xYAZBRcjXHXGqVeYZxS6KsTHGcsDLHdHetH7YYp7pTClye\nky4okybnlc9p8nUGYnq8zinO3XTbRR61plJts/MiRTzZYjkIauT3WxpNqeM3matpvS51/Hbb99Fi\nLszjkztEu7G2d7FdPPpTUVe8+o5eOTfi9zI0+Sh3wY6Rj/K0d9wUnNNu0DynofbBRiL41yt2IXQX\ngdR53JzH9fqYWy4/jG0uDObCm0bU3wPg+92HuwDg/zjn/o6IfgHgu0T0MIA3AXxmoCsbDIYtw6oL\n3zn3OoDb+3x+EcB98TMMBsPljqF67lWWl/Dcz34CADj35nFRl2OykDaP8XRMDWZeopziLmNkHg0l\nCuVYJFmDmbaihhRRm9u9SKnJGhwT4XOsj4biCJTehVKkLDCVo1SW/VOOmcAYGUaxpMLimIdiQ5nR\nwNQYYmKvduzi6siy4rqvVbwZsFplpj01H8tVL+o3laeai/zx+Iif+9Gy9IYssTFGF95Q4/CpsaMr\nbuyVyzv3iXaFIjOzKhdCQbDBPtemMk4+4nJ66yudqU+mzJbt0kbTCc0wpHEokkNLk20wGFaFLXyD\nIYOwhW8wZBBD1fHrlQpef7HLrqN0wjwjfCwpHZ9zwks9M1kJqqs0xTw6j+fRW1yuinZTy15nHhuV\nujtPnsyjz5ziSc8J4kaVs67NzouN0V+v0fL68+iY5FDnPI4txeZSWfL3I/IMKPtSi83j3LyMraqz\nfYMRtr9QU5FvFabjt7XZkrkZX73vyl65qcaRZ27A2gesNetzKCye8Zzyo3sPinbjV7+7Vy7vkA6k\nPBrSBUxq/OEZxPk1WXfXewj9SVD79div2DnktklVZ7z6BoNhNdjCNxgyiOEScbjIpy3W3ldMJC4r\nwgEhxTAiSNLeV9yEp2Qf7v1XZ+LxQkOqHK+dvdgrR0os3T3lRe4R5hlISpwnHk2nTI5gaZULRXmf\nTeZFOL/oTWzFslRHcsS9BuUc1Fgq6BpLV6Wj89psVs/PL4i6FhO/R8Z8dFuT5H0ix6IL1ffZXvaE\nmPkLPgKvrUg/to97BYqUylRh0YU8TVb+okwfXV/0Xp/1SSnql3Zf3SuP7/IqR2FCuY8z/UlLzf2j\nUPog8PxpT740iI0joAYMatCzN77BkEHYwjcYMoihivpEhHyXZ15nb+WZXbUIz8WmEstSGylxiouz\nOnCG74hWmTjcUDvrb15iQSPLdVG3f7evm2Qiqk7llWdWiaYKAhKppZTYy3e8T5/3onJRidHjjCxE\nZ/TlG/k89RYVJOnH+LQPkd5+83tE3Y6pXb68y7cbGZOkH0WmqujAmdm3vTh+6pVf9sq/Pv22aDc5\n6vsYVUFRY0yd2ln219Y72MSChaonj4m6xRPeQ7TCxPvi9p3yWld4fruxPZLrrrh92l9LEatILr1k\ncTscpJPQR+wZTj6ybLkGg2FV2MI3GDIIW/gGQwYxZB3fR0tpMkwI7nIVFcf0Hu4JF6kIpZZIM61I\nLqL+vOMNRYZRZccn5mTU2gWWqnly3EeZbRuTEWfE8uU1c/I+S9umeuWJPdeKunGmS15xhc8HV12S\n5jZOXqm51sfHPS/7BOtvYse0aDex3RNijIyMiroi20cpBHLKce/FvPK2PHDDzb3yVYd8ZN3smVOi\n3YlXXuiVXzv+iqjbUfLvpTYzW26P5Hh5hF+rLr/PMpv/Es8VPnNGtKvMeC/B2nGZ16G813P/T95y\nl6jLCUJQ/7mO/guq4El7AwO44614EKaNArQ3vsGQQdjCNxgyiOGK+qBeOuKySsckyA80hx0Tvxst\nLs5LU5wwZSlPNS7qR0wc0qQfPF1yXakBCyzd8/Te63rl3Qevl+NgA8mrFFej2yYT6xz7HS5PMNFc\neefl2PxoEZt7QBaYqY9Uu3qdE2xI8bDA1DCukum5Smqnx8VVh137rhbt+PHsbYdF3ZssrdrJMyd6\n5TMX50W7UfYsjSjz5vQoMwM22PeuuPnKLGcC1aWn5MIxJvpPTIm6HTfc5vsX4r1WLZNd9xKDdjSv\nXgK/31pgb3yDIYOwhW8wZBC28A2GDGLo5rwVl1utmwozndNmuv5pirVqFAk9XtUJ/cj3N6LcbWtM\nrS8ohsp9193UKx96z52+nSLDbLf9xbTJscm46etN6RIsiRZCUYjJ/PD8mOv7eaXTFgveHbZQ1Psc\nRVbmJjtlzuN6vK7jqcKZGbBQVHsBbIwT0zLT0q0fur9XrlU9v//8Wen2e4655b791muibnHJuz5v\nZ3p8Qc8bm/pyWdWxeykpMhKpnwdMacmp/5KhI/yCTd1Afdsb32DIIGzhGwwZxHCJOIiYKKp477np\nSckruRxLs8zE3Ii0KMTTQvX31FsZxwoayuxXZ2rFlIrSOnjjLb4LNt5GXYp/nN8vRoDBRH/ntHdX\n/wjFkKivo/NywpuOpdpWZCHckbGt5oofclWlqLncWUNtkuLciG02jraKVuRqQBQpNYDdW4ml0Np3\n6CbR7sD1nnOvuii9HC+efrNXvsS8BhcWLol2daZK5FU68APXHOqVJ68+JOoSifADpjiNMAdfwjnr\ntOeleuMT0Q4i+isi+jURvUpEHyCiaSJ6koiOdf9Ord6TwWC4HJBW1P8agL9zzt2ETjqtVwF8EcBT\nzrnrATzVPTYYDO8ApMmWux3AhwD8WwBwzjUANIjoQQD3dps9BuDHAL6w6hW7YqombhBiozqFe6rx\nLLLtlt79TxaxeTqsKiPfWFRBHVT2HHNXMxESkNx61ZqnoOa7+HockRqHkJaVRx4Yp12eBZdoqju+\nQx8TErk1QMybvBYXo2PqAk8BFqAK58eaWEWoHKF2It2YUv/y3ELB1Zvke9m+c5eom77iCt/f+3iA\njVJbmJempvnmBCSFgiQLcXy++efhqJxUVTHOvYBlYDNSaB0EcB7A/yai54jof3XTZe9xzq2EOJ1F\nJ6uuwWB4ByDNwi8AeC+Arzvn7gSwDCXWu87uRN8fHSJ6hIiOENGRSr3Vr4nBYBgy0iz8UwBOOeee\n6R7/FTo/BOeIaC8AdP/O9DvZOfeoc+6wc+7wWHm4RgSDwdAfq65E59xZIjpJRDc6544CuA/AK91/\nDwH4cvfv42kuuGI6yimiSS4v6HTMwkzHPo8Cek5LRdbVWHrtCis3nZyC3Xt9tFihLAkqlyuebJPv\nLyg1Xnge6jEKs6WaAm5y43sgOpJMEGUogkoZTcei51S7EiOyLMQ88vh5ydF5wqtP6f/cM5P3p82K\nop2u45GG+f4eiYD0wot5MvI9ilz/vSJ9nt5/ogQ9HkAiOWZaQoxYH8FmfP9mfUp+2lfwfwDwLSIq\nAXgdwL9DR1r4LhE9DOBNAJ8Z7NIGg2GrkGrhO+eeB3C4T9V9Gzscg8EwDAw5hRa8rK4llYh7gck6\nbpqLomRvMX4kBX1JqsEteKVxmYl2hHGvLy4uiTpJWMHEciWzc6++mAmMm9GUiM3F8RLjrC8qXj3R\nriRFeG5uKpW4yC774CmpdOCMCKpJINTQdVpdSAoQircLeSGy83L9RfbOcXIdJZg3dd4FcRhLwpxs\nR0vkxNcQ6bWSm4Ukdu7Rqi81qPef+eobDBmELXyDIYOwhW8wZBBDjs5D76dGqyScDFNz4HOOfKHv\nqz6arB033wFAnZnfHLGUy6PbRLtKlZFjkIy6I5aeOp/nOrLSs7mOr91Lmd6qySt4XsAy092LSo/n\nOr6uKwn9v9T3HH2c1pxXjJFtsnkMkH7mA6a4sCtuOlNcWHdP0M+DrBb6MBDpmXRaLDov+dpJpr8w\nN7+OlEwzQg974xsMGYQtfIMhg6C1kACs+WJE59Fx9tkF4MLQLtwfl8MYABuHho1DYtBxXOOc271a\no6Eu/N5FiY445/o5BGVqDDYOG8dWjcNEfYMhg7CFbzBkEFu18B/doutyXA5jAGwcGjYOiU0Zx5bo\n+AaDYWthor7BkEEMdeET0QNEdJSIjhPR0Fh5ieibRDRDRC+xz4ZOD05EB4joaSJ6hYheJqLPb8VY\niGiEiH5ORC90x/Gn3c8PEtEz3e/nO13+hU0HEeW7fI4/2KpxENEJIvoVET1PREe6n23FMzIUKvuh\nLXwiygP4nwA+BuBmAJ8lopuHdPm/APCA+mwr6MFbAP7IOXczgLsBfK47B8MeSx3Ah51ztwO4A8AD\nRHQ3gK8A+Kpz7hCAOQAPb/I4VvB5dCjbV7BV4/gd59wdzHy2Fc/IcKjsnXND+QfgAwD+nh1/CcCX\nhnj9awG8xI6PAtjbLe8FcHRYY2FjeBzAR7dyLADGAPwSwPvRcRQp9Pu+NvH6+7sP84cB/ACdiI6t\nGMcJALvUZ0P9XgBsB/AGuntvmzmOYYr6VwE4yY5PdT/bKmwpPTgRXQvgTgDPbMVYuuL18+iQpD4J\n4DUA8865FSrkYX0/fw7gj+EpWnZu0TgcgB8R0bNE9Ej3s2F/L0OjsrfNPYTpwTcDRLQNwF8D+EPn\nnEj2NqyxOOfazrk70Hnj3gXgplVO2XAQ0ScBzDjnnh32tfvgHufce9FRRT9HRB/ilUP6XtZFZT8I\nhrnwTwM4wI73dz/bKqSiB99oEFERnUX/Lefc32zlWADAOTcP4Gl0ROod5GOPh/H9fBDAp4joBIBv\noyPuf20LxgHn3Onu3xkA30fnx3DY38u6qOwHwTAX/i8AXN/dsS0B+D0ATwzx+hpPoEMLDgxAD74e\nUCc4/BsAXnXO/dlWjYWIdhPRjm55FJ19hlfR+QH49LDG4Zz7knNuv3PuWnSeh390zv3BsMdBRONE\nNLFSBnA/gJcw5O/FOXcWwEkiurH70QqV/caPY7M3TdQmxccB/AYdffK/DPG6fwngDIAmOr+qD6Oj\nSz4F4BiAfwAwPYRx3IOOmPYigOe7/z4+7LEAuA3Ac91xvATgv3Y/vw7AzwEcB/A9AOUhfkf3AvjB\nVoyje70Xuv9eXnk2t+gZuQPAke5387cApjZjHOa5ZzBkELa5ZzBkELbwDYYMwha+wZBB2MI3GDII\nW/gGQwZhC99gyCBs4RsMGYQtfIMhg/j/CVqDcFE3+QsAAAAASUVORK5CYII=\n",
       "text": [
        "<matplotlib.figure.Figure at 0x7fcbb6aa8e10>"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u548c\u5f80\u5e38\u4e00\u6837\uff0c\u5c06\u56fe\u50cf\u6570\u636e\u96c6\u5c55\u5f00\u53d8\u5e73\uff0c\u7136\u540e\u9664\u4ee5255\u8fdb\u884c\u5f52\u4e00\u5316\u3002\u6700\u91cd\u8981\u7684\u662f\uff0c\u4f60\u8981\u50cf\u56fe\u4e00\u6240\u793a\uff0c\u628a\u6bcf\u4e2a\u6807\u7b7e\u8f6c\u6362\u4e3a\u4e00\u4e2a\u70ed\u70b9\u5411\u91cf(a one-hot vector). \u8fd0\u884c\u4e0b\u9762\u7684\u5355\u5143\u683c\u6765\u6267\u884c\u6b64\u64cd\u4f5c"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# Flatten the training and test images\n",
      "X_train_flatten = X_train_orig.reshape(X_train_orig.shape[0], -1).T\n",
      "X_test_flatten = X_test_orig.reshape(X_test_orig.shape[0], -1).T\n",
      "# Normalize image vectors\n",
      "X_train = X_train_flatten/255.\n",
      "X_test = X_test_flatten/255.\n",
      "# Convert training and test labels to one hot matrices\n",
      "Y_train = convert_to_one_hot(Y_train_orig, 6)\n",
      "Y_test = convert_to_one_hot(Y_test_orig, 6)\n",
      "\n",
      "print (\"number of training examples = \" + str(X_train.shape[1]))\n",
      "print (\"number of test examples = \" + str(X_test.shape[1]))\n",
      "print (\"X_train shape: \" + str(X_train.shape))\n",
      "print (\"Y_train shape: \" + str(Y_train.shape))\n",
      "print (\"X_test shape: \" + str(X_test.shape))\n",
      "print (\"Y_test shape: \" + str(Y_test.shape))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "number of training examples = 1080\n",
        "number of test examples = 120\n",
        "X_train shape: (12288, 1080)\n",
        "Y_train shape: (6, 1080)\n",
        "X_test shape: (12288, 120)\n",
        "Y_test shape: (6, 120)\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**\u8bf7\u6ce8\u610f** that 12288 comes from $64 \\times 64 \\times 3$. \u6bcf\u4e2a\u56fe\u50cf\u662f64* 64 \u50cf\u7d20\u7684\u6b63\u65b9\u5f62 , 3 \u662f RGB \u989c\u8272"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**\u60a8\u7684\u76ee\u6807**\u662f\u5efa\u7acb\u4e00\u4e2a\u80fd\u591f\u9ad8\u51c6\u786e\u5ea6\u8bc6\u522b\u7b26\u53f7\u7684\u7b97\u6cd5\u3002\u8981\u505a\u5230\u8fd9\u4e00\u70b9\uff0c\u4f60\u8981\u5efa\u7acb\u4e00\u4e2atensorflow\u6a21\u578b\uff0c\u8fd9\u4e2a\u6a21\u578b\u548c\u4f60\u4e4b\u524d\u7528 python \u5b9e\u73b0\u7684\u732b\u54aa\u8bc6\u522b\u6a21\u578b\u51e0\u4e4e\u4e00\u6837(\u4e0d\u540c\u7684\u662f\u73b0\u5728\u4f7f\u7528softmax\u8f93\u51fa). \u8fd9\u662f\u4e00\u4e2a\u5f88\u597d\u7684\u573a\u5408\u53bb\u5bf9\u6bd4\u4e24\u8005\u5b9e\u73b0\u7684\u5dee\u5f02\uff08numpy\u548ctensorflow\uff09\u3002\n",
      "\n",
      "**The model** is *LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX*. SIGMOID\u8f93\u51fa\u5c42\u5df2\u88ab\u8f6c\u6362\u4e3aSOFTMAX. A SOFTMAX layer generalizes SIGMOID to when there are more than two classes. "
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.1 - Create placeholders\n",
      "\n",
      "\u4f60\u7684\u7b2c\u4e00\u4e2a\u4efb\u52a1\u662f\u4e3a`X` \u548c `Y`\u521b\u5efa\u5360\u4f4d\u7b26. \u8fd9\u5c06\u5141\u8bb8\u60a8\u7a0d\u540e\u5728\u8fd0\u884c\u4f1a\u8bdd\u65f6\u4f20\u9012\u60a8\u7684\u8bad\u7ec3\u6570\u636e\n",
      "\n",
      "**Exercise:** \u6267\u884c\u4e0b\u9762\u7684\u51fd\u6570\u53bb\u521b\u5efatensorflow\u4e2d\u7684\u5360\u4f4d\u7b26"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: create_placeholders\n",
      "\n",
      "def create_placeholders(n_x, n_y):\n",
      "    \"\"\"\n",
      "    Creates the placeholders for the tensorflow session.\n",
      "    \n",
      "    Arguments:\n",
      "    n_x -- scalar, size of an image vector (num_px * num_px = 64 * 64 * 3 = 12288)\n",
      "    n_y -- scalar, number of classes (from 0 to 5, so -> 6)\n",
      "    \n",
      "    Returns:\n",
      "    X -- placeholder for the data input, of shape [n_x, None] and dtype \"float\"\n",
      "    Y -- placeholder for the input labels, of shape [n_y, None] and dtype \"float\"\n",
      "    \n",
      "    Tips:\n",
      "    - You will use None because it let's us be flexible on the number of examples you will for the placeholders.\n",
      "      In fact, the number of examples during test/train is different.\n",
      "    \"\"\"\n",
      "\n",
      "    ### START CODE HERE ### (approx. 2 lines)\n",
      "    X = tf.placeholder(tf.float32, shape=(n_x,None), name=\"X\")\n",
      "    Y = tf.placeholder(tf.float32, shape=(n_y,None), name=\"Y\")\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return X, Y"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "X, Y = create_placeholders(12288, 6)\n",
      "print (\"X = \" + str(X))\n",
      "print (\"Y = \" + str(Y))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "X = Tensor(\"X:0\", shape=(12288, ?), dtype=float32)\n",
        "Y = Tensor(\"Y:0\", shape=(6, ?), dtype=float32)\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**: \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **X**\n",
      "        </td>\n",
      "        <td>\n",
      "        Tensor(\"Placeholder_1:0\", shape=(12288, ?), dtype=float32) (not necessarily Placeholder_1)\n",
      "        </td>\n",
      "    </tr>\n",
      "    <tr> \n",
      "        <td>\n",
      "            **Y**\n",
      "        </td>\n",
      "        <td>\n",
      "        Tensor(\"Placeholder_2:0\", shape=(10, ?), dtype=float32) (not necessarily Placeholder_2)\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.2 - Initializing the parameters\n",
      "\n",
      "\u4f60\u7684\u7b2c\u4e8c\u4e2a\u4efb\u52a1\u662f\u521d\u59cb\u5316tensorflow\u7684\u53c2\u6570\u3002\n",
      "\n",
      "**Exercise:** \u6267\u884c\u4e0b\u9762\u7684\u51fd\u6570\u6765\u521d\u59cb\u5316tensorflow\u4e2d\u7684\u53c2\u6570\u3002You are going use Xavier Initialization for weights and Zero Initialization for biases. The shapes are given below. As an example, to help you, for W1 and b1 you could use: \n",
      "\n",
      "```python\n",
      "W1 = tf.get_variable(\"W1\", [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
      "b1 = tf.get_variable(\"b1\", [25,1], initializer = tf.zeros_initializer())\n",
      "```\n",
      "\u8bf7\u4f7f\u7528 `seed = 1` \uff0c\u4ee5\u786e\u4fdd\u60a8\u7684\u7ed3\u679c\u548c\u6211\u4eec\u7684\u76f8\u5339\u914d\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: initialize_parameters\n",
      "\n",
      "def initialize_parameters():\n",
      "    \"\"\"\n",
      "    Initializes parameters to build a neural network with tensorflow. The shapes are:\n",
      "                        W1 : [25, 12288]\n",
      "                        b1 : [25, 1]\n",
      "                        W2 : [12, 25]\n",
      "                        b2 : [12, 1]\n",
      "                        W3 : [6, 12]\n",
      "                        b3 : [6, 1]\n",
      "    \n",
      "    Returns:\n",
      "    parameters -- a dictionary of tensors containing W1, b1, W2, b2, W3, b3\n",
      "    \"\"\"\n",
      "    \n",
      "    tf.set_random_seed(1)                   # so that your \"random\" numbers match ours\n",
      "        \n",
      "    ### START CODE HERE ### (approx. 6 lines of code)\n",
      "    W1 = tf.get_variable(\"W1\", [25,12288], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
      "    b1 = tf.get_variable(\"b1\", [25,1], initializer = tf.zeros_initializer())\n",
      "    W2 = tf.get_variable(\"W2\", [12,25], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
      "    b2 = tf.get_variable(\"b2\", [12,1], initializer = tf.zeros_initializer())\n",
      "    W3 = tf.get_variable(\"W3\", [6,12], initializer = tf.contrib.layers.xavier_initializer(seed = 1))\n",
      "    b3 = tf.get_variable(\"b3\", [6,1], initializer = tf.zeros_initializer())\n",
      "    ### END CODE HERE ###\n",
      "\n",
      "    parameters = {\"W1\": W1,\n",
      "                  \"b1\": b1,\n",
      "                  \"W2\": W2,\n",
      "                  \"b2\": b2,\n",
      "                  \"W3\": W3,\n",
      "                  \"b3\": b3}\n",
      "    \n",
      "    return parameters"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tf.reset_default_graph()\n",
      "with tf.Session() as sess:\n",
      "    parameters = initialize_parameters()\n",
      "    print(\"W1 = \" + str(parameters[\"W1\"]))\n",
      "    print(\"b1 = \" + str(parameters[\"b1\"]))\n",
      "    print(\"W2 = \" + str(parameters[\"W2\"]))\n",
      "    print(\"b2 = \" + str(parameters[\"b2\"]))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "W1 = <tf.Variable 'W1:0' shape=(25, 12288) dtype=float32_ref>\n",
        "b1 = <tf.Variable 'b1:0' shape=(25, 1) dtype=float32_ref>\n",
        "W2 = <tf.Variable 'W2:0' shape=(12, 25) dtype=float32_ref>\n",
        "b2 = <tf.Variable 'b2:0' shape=(12, 1) dtype=float32_ref>\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**: \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **W1**\n",
      "        </td>\n",
      "        <td>\n",
      "         < tf.Variable 'W1:0' shape=(25, 12288) dtype=float32_ref >\n",
      "        </td>\n",
      "    </tr>\n",
      "    <tr> \n",
      "        <td>\n",
      "            **b1**\n",
      "        </td>\n",
      "        <td>\n",
      "        < tf.Variable 'b1:0' shape=(25, 1) dtype=float32_ref >\n",
      "        </td>\n",
      "    </tr>\n",
      "    <tr> \n",
      "        <td>\n",
      "            **W2**\n",
      "        </td>\n",
      "        <td>\n",
      "        < tf.Variable 'W2:0' shape=(12, 25) dtype=float32_ref >\n",
      "        </td>\n",
      "    </tr>\n",
      "    <tr> \n",
      "        <td>\n",
      "            **b2**\n",
      "        </td>\n",
      "        <td>\n",
      "        < tf.Variable 'b2:0' shape=(12, 1) dtype=float32_ref >\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "As expected, the parameters haven't been evaluated yet."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.3 - tensorflow \u7684\u524d\u5411\u4f20\u64ad\n",
      "\n",
      "\u60a8\u73b0\u5728\u5c06\u5b9e\u73b0tensorflow\u4e2d\u7684\u524d\u5411\u4f20\u64ad\u6a21\u5757\u3002\u8be5\u51fd\u6570\u5c06\u5e26\u6709\u4e00\u4e2a\u53c2\u6570\u5b57\u5178\uff0c\u5b83\u5c06\u5b8c\u6210\u524d\u5411\u4f20\u9012\u3002\u60a8\u5c06\u4f7f\u7528\u7684\u529f\u80fd\u662f\uff1a\n",
      "\n",
      "- `tf.add(...,...)` to do an addition\n",
      "- `tf.matmul(...,...)` to do a matrix multiplication\n",
      "- `tf.nn.relu(...)` to apply the ReLU activation\n",
      "\n",
      "**Question:** \u5b9e\u73b0\u795e\u7ecf\u7f51\u7edc\u7684\u524d\u5411\u4f20\u9012. We commented for you the numpy equivalents so that you can compare the tensorflow implementation to numpy. It is important to note that the forward propagation stops at `z3`. The reason is that in tensorflow the last linear layer output is given as input to the function computing the loss. Therefore, you don't need `a3`!\n",
      "\n"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: forward_propagation\n",
      "\n",
      "def forward_propagation(X, parameters):\n",
      "    \"\"\"\n",
      "    Implements the forward propagation for the model: LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SOFTMAX\n",
      "    \n",
      "    Arguments:\n",
      "    X -- input dataset placeholder, of shape (input size, number of examples)\n",
      "    parameters -- python dictionary containing your parameters \"W1\", \"b1\", \"W2\", \"b2\", \"W3\", \"b3\"\n",
      "                  the shapes are given in initialize_parameters\n",
      "\n",
      "    Returns:\n",
      "    Z3 -- the output of the last LINEAR unit\n",
      "    \"\"\"\n",
      "    \n",
      "    # Retrieve the parameters from the dictionary \"parameters\" \n",
      "    W1 = parameters['W1']\n",
      "    b1 = parameters['b1']\n",
      "    W2 = parameters['W2']\n",
      "    b2 = parameters['b2']\n",
      "    W3 = parameters['W3']\n",
      "    b3 = parameters['b3']\n",
      "    \n",
      "    ### START CODE HERE ### (approx. 5 lines)              # Numpy Equivalents:\n",
      "    Z1 = tf.matmul(W1, X)+b1                               # Z1 = np.dot(W1, X) + b1\n",
      "    A1 = tf.nn.relu(Z1)                                    # A1 = relu(Z1)\n",
      "    Z2 = tf.matmul(W2, A1)+b2                              # Z2 = np.dot(W2, a1) + b2\n",
      "    A2 = tf.nn.relu(Z2)                                    # A2 = relu(Z2)\n",
      "    Z3 = tf.matmul(W3, A2)+b3                              # Z3 = np.dot(W3,Z2) + b3\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return Z3"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "tf.reset_default_graph()\n",
      "\n",
      "with tf.Session() as sess:\n",
      "    X, Y = create_placeholders(12288, 6)\n",
      "    parameters = initialize_parameters()\n",
      "    Z3 = forward_propagation(X, parameters)\n",
      "    print(\"Z3 = \" + str(Z3))"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Z3 = Tensor(\"add_2:0\", shape=(6, ?), dtype=float32)\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**: \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **Z3**\n",
      "        </td>\n",
      "        <td>\n",
      "        Tensor(\"Add_2:0\", shape=(6, ?), dtype=float32)\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "\u60a8\u53ef\u80fd\u5df2\u7ecf\u6ce8\u610f\u5230\uff0c\u524d\u5411\u4f20\u64ad\u4e0d\u4f1a\u8f93\u51fa\u4efb\u4f55\u7f13\u5b58. You will understand why below, when we get to brackpropagation."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.4 Compute cost\n",
      "\n",
      "\u5c31\u50cf\u4e4b\u524d\u770b\u5230\u7684\uff0c\u6211\u4eec\u4f7f\u7528\u4e0b\u9762\u7684\u65b9\u6cd5\u8ba1\u7b97\u6210\u672c\u975e\u5e38\u7b80\u5355\uff1a\n",
      "```python\n",
      "tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = ..., labels = ...))\n",
      "```\n",
      "**Question**: \u5b9e\u73b0\u4e0b\u9762\u7684 cost function. \n",
      "- It is important to know that the \"`logits`\" and \"`labels`\" inputs of `tf.nn.softmax_cross_entropy_with_logits` are expected to be of shape (number of examples, num_classes). We have thus transposed Z3 and Y for you.\n",
      "- Besides, `tf.reduce_mean` basically does the summation over the examples."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "# GRADED FUNCTION: compute_cost \n",
      "\n",
      "def compute_cost(Z3, Y):\n",
      "    \"\"\"\n",
      "    Computes the cost\n",
      "    \n",
      "    Arguments:\n",
      "    Z3 -- output of forward propagation (output of the last LINEAR unit), of shape (6, number of examples)\n",
      "    Y -- \"true\" labels vector placeholder, same shape as Z3\n",
      "    \n",
      "    Returns:\n",
      "    cost - Tensor of the cost function\n",
      "    \"\"\"\n",
      "    \n",
      "    # to fit the tensorflow requirement for tf.nn.softmax_cross_entropy_with_logits(...,...)\n",
      "    logits = tf.transpose(Z3)\n",
      "    labels = tf.transpose(Y)\n",
      "    \n",
      "    ### START CODE HERE ### (1 line of code)\n",
      "    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = logits, labels = labels))\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    return cost"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": null
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "tf.reset_default_graph()\n",
      "\n",
      "with tf.Session() as sess:\n",
      "    X, Y = create_placeholders(12288, 6)\n",
      "    parameters = initialize_parameters()\n",
      "    Z3 = forward_propagation(X, parameters)\n",
      "    cost = compute_cost(Z3, Y)\n",
      "    print(\"cost = \" + str(cost))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": null
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**: \n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **cost**\n",
      "        </td>\n",
      "        <td>\n",
      "        Tensor(\"Mean:0\", shape=(), dtype=float32)\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>"
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.5 - Backward propagation & parameter updates\n",
      "\n",
      "\u6240\u6709\u7684\u53cd\u5411\u4f20\u64ad\u548c\u53c2\u6570\u66f4\u65b0\u90fd\u57281\u884c\u4ee3\u7801\u4e2d\u5904\u7406\u3002\n",
      "\n",
      "\u8ba1\u7b97\u6210\u672c\u51fd\u6570\u540e\u3002\u4f60\u5c06\u521b\u5efa\u4e00\u4e2a\"`optimizer`\"\u5bf9\u8c61. \u5728\u8fd0\u884ctf.session\u65f6\uff0c\u4f60\u5fc5\u987b\u8c03\u7528\u8fd9\u4e2a\u5bf9\u8c61\u548c\u6210\u672c\u51fd\u6570. \u5f53\u88ab\u8c03\u7528\u65f6\uff0c\u5b83\u5c06\u6839\u636e\u6240\u9009\u62e9\u7684\u65b9\u6cd5\u548c\u5b66\u4e60\u7387\u5bf9\u7ed9\u5b9a\u7684\u6210\u672c\u8fdb\u884c\u4f18\u5316\u3002\n",
      "\n",
      "\u4f8b\u5982\uff0c\u5bf9\u4e8e\u68af\u5ea6\u4e0b\u964d\uff0c\u4f18\u5316\u5668\u5c06\u662f\uff1a\n",
      "```python\n",
      "optimizer = tf.train.GradientDescentOptimizer(learning_rate = learning_rate).minimize(cost)\n",
      "```\n",
      "\n",
      "\u8981\u8fdb\u884c\u4f18\u5316\uff0c\u60a8\u53ef\u4ee5\u8fd9\u6837\u505a\uff1a\n",
      "```python\n",
      "_ , c = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})\n",
      "```\n",
      "\n",
      "This computes the backpropagation by passing through the tensorflow graph in the reverse order. From cost to inputs.\n",
      "\n",
      "**Note** \u5728\u7f16\u7801\u65f6\uff0c\u6211\u4eec\u7ecf\u5e38\u4f7f\u7528\u201c\u4e00\u6b21\u6027\u201d\u53d8\u91cf \u4e0b\u6ed1\u7ebf `_` \u6765\u5b58\u50a8\u6211\u4eec\u7a0d\u540e\u4e0d\u9700\u8981\u4f7f\u7528\u7684\u503c\u3002 `_` takes on the evaluated value of `optimizer`, which we don't need (and `c` takes the value of the `cost` variable). "
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.6 - Building the model\n",
      "\n",
      "Now, you will bring it all together! \n",
      "\n",
      "**Exercise:** \u5b8c\u6210\u6a21\u578b\u3002\u4f60\u5c06\u4f1a\u8c03\u7528\u4f60\u4e4b\u524d\u5df2\u7ecf\u5b9e\u73b0\u7684\u529f\u80fd\u3002"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "def model(X_train, Y_train, X_test, Y_test, learning_rate = 0.0001,\n",
      "          num_epochs = 1500, minibatch_size = 32, print_cost = True):\n",
      "    \"\"\"\n",
      "    Implements a three-layer tensorflow neural network: LINEAR->RELU->LINEAR->RELU->LINEAR->SOFTMAX.\n",
      "    \n",
      "    Arguments:\n",
      "    X_train -- training set, of shape (input size = 12288, number of training examples = 1080)\n",
      "    Y_train -- test set, of shape (output size = 6, number of training examples = 1080)\n",
      "    X_test -- training set, of shape (input size = 12288, number of training examples = 120)\n",
      "    Y_test -- test set, of shape (output size = 6, number of test examples = 120)\n",
      "    learning_rate -- learning rate of the optimization\n",
      "    num_epochs -- number of epochs of the optimization loop\n",
      "    minibatch_size -- size of a minibatch\n",
      "    print_cost -- True to print the cost every 100 epochs\n",
      "    \n",
      "    Returns:\n",
      "    parameters -- parameters learnt by the model. They can then be used to predict.\n",
      "    \"\"\"\n",
      "    \n",
      "    ops.reset_default_graph()                         # to be able to rerun the model without overwriting tf variables\n",
      "    tf.set_random_seed(1)                             # to keep consistent results\n",
      "    seed = 3                                          # to keep consistent results\n",
      "    (n_x, m) = X_train.shape                          # (n_x: input size, m : number of examples in the train set)\n",
      "    n_y = Y_train.shape[0]                            # n_y : output size\n",
      "    costs = []                                        # To keep track of the cost\n",
      "    \n",
      "    # Create Placeholders of shape (n_x, n_y)\n",
      "    ### START CODE HERE ### (1 line)\n",
      "    X, Y = create_placeholders(n_x, n_y)\n",
      "    ### END CODE HERE ###\n",
      "\n",
      "    # Initialize parameters\n",
      "    ### START CODE HERE ### (1 line)\n",
      "    parameters = initialize_parameters()\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    # Forward propagation: Build the forward propagation in the tensorflow graph\n",
      "    ### START CODE HERE ### (1 line)\n",
      "    Z3 = forward_propagation(X, parameters)\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    # Cost function: Add cost function to tensorflow graph\n",
      "    ### START CODE HERE ### (1 line)\n",
      "    cost = compute_cost(Z3, Y)\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer.\n",
      "    ### START CODE HERE ### (1 line)\n",
      "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)\n",
      "    ### END CODE HERE ###\n",
      "    \n",
      "    # Initialize all the variables\n",
      "    init = tf.global_variables_initializer()\n",
      "\n",
      "    # Start the session to compute the tensorflow graph\n",
      "    with tf.Session() as sess:\n",
      "        \n",
      "        # Run the initialization\n",
      "        sess.run(init)\n",
      "        \n",
      "        # Do the training loop\n",
      "        for epoch in range(num_epochs):\n",
      "\n",
      "            epoch_cost = 0.                       # Defines a cost related to an epoch\n",
      "            num_minibatches = int(m / minibatch_size) # number of minibatches of size minibatch_size in the train set\n",
      "            seed = seed + 1\n",
      "            minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)\n",
      "\n",
      "            for minibatch in minibatches:\n",
      "\n",
      "                # Select a minibatch\n",
      "                (minibatch_X, minibatch_Y) = minibatch\n",
      "                \n",
      "                # IMPORTANT: The line that runs the graph on a minibatch.\n",
      "                # Run the session to execute the \"optimizer\" and the \"cost\", the feedict should contain a minibatch for (X,Y).\n",
      "                ### START CODE HERE ### (1 line)\n",
      "                _ , minibatch_cost = sess.run([optimizer, cost], feed_dict={X: minibatch_X, Y: minibatch_Y})\n",
      "                ### END CODE HERE ###\n",
      "                \n",
      "                epoch_cost += minibatch_cost / num_minibatches\n",
      "\n",
      "            # Print the cost every epoch\n",
      "            if print_cost == True and epoch % 100 == 0:\n",
      "                print (\"Cost after epoch %i: %f\" % (epoch, epoch_cost))\n",
      "            if print_cost == True and epoch % 5 == 0:\n",
      "                costs.append(epoch_cost)\n",
      "                \n",
      "        # plot the cost\n",
      "        plt.plot(np.squeeze(costs))\n",
      "        plt.ylabel('cost')\n",
      "        plt.xlabel('iterations (per tens)')\n",
      "        plt.title(\"Learning rate =\" + str(learning_rate))\n",
      "        plt.show()\n",
      "\n",
      "        # lets save the parameters in a variable\n",
      "        parameters = sess.run(parameters)\n",
      "        print (\"Parameters have been trained!\")\n",
      "\n",
      "        # Calculate the correct predictions\n",
      "        correct_prediction = tf.equal(tf.argmax(Z3), tf.argmax(Y))\n",
      "\n",
      "        # Calculate accuracy on the test set\n",
      "        accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
      "\n",
      "        print (\"Train Accuracy:\", accuracy.eval({X: X_train, Y: Y_train}))\n",
      "        print (\"Test Accuracy:\", accuracy.eval({X: X_test, Y: Y_test}))\n",
      "        \n",
      "        return parameters"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": null
    },
    {
     "cell_type": "markdown",
     "metadata": {
      "collapsed": true
     },
     "source": [
      "\u8fd0\u884c\u4e0b\u9762\u7684\u5355\u5143\u683c\u6765\u8bad\u7ec3\u4f60\u7684\u6a21\u578b\uff01\u5728\u6211\u4eec\u7684\u673a\u5668\u4e0a\u5927\u7ea6\u9700\u89815\u5206\u949f. Your \"Cost after epoch 100\" should be 1.016458. \u5982\u679c\u4e0d\u662f\uff0c\u4e0d\u8981\u6d6a\u8d39\u65f6\u95f4; \u901a\u8fc7\u70b9\u51fb notebook \u4e0a\u65b9\u7684\u65b9\u5757 (\u2b1b)\u53bb\u4e2d\u65ad\u8bad\u7ec3, \u5e76\u5c1d\u8bd5\u66f4\u6b63\u60a8\u7684\u4ee3\u7801. \u5982\u679c\u635f\u5931\u662f\u6b63\u786e\u7684\uff0c\u8bf7\u4f11\u606f\u4e00\u4e0b\uff0c5\u5206\u949f\u540e\u56de\u6765\uff01"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "parameters = model(X_train, Y_train, X_test, Y_test)"
     ],
     "language": "python",
     "metadata": {
      "scrolled": false
     },
     "outputs": [],
     "prompt_number": null
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "**Expected Output**:\n",
      "\n",
      "<table> \n",
      "    <tr> \n",
      "        <td>\n",
      "            **Train Accuracy**\n",
      "        </td>\n",
      "        <td>\n",
      "        0.999074\n",
      "        </td>\n",
      "    </tr>\n",
      "    <tr> \n",
      "        <td>\n",
      "            **Test Accuracy**\n",
      "        </td>\n",
      "        <td>\n",
      "        0.716667\n",
      "        </td>\n",
      "    </tr>\n",
      "\n",
      "</table>\n",
      "\n",
      "\u592a\u4ee4\u4eba\u60ca\u8bb6\u4e86\uff0c\u60a8\u7684\u7b97\u6cd5\u53ef\u4ee5\u8bc6\u522b\u4ee3\u88680\u52305\u4e4b\u95f4\u7684\u6570\u5b57\u7684\u7b26\u53f7\uff0c\u51c6\u786e\u7387\u4e3a71.7\uff05\u3002\n",
      "\n",
      "**Insights**:\n",
      "- \u4f60\u7684\u6a21\u578b\u4f3c\u4e4e\u8db3\u591f\u5927\u53bb\u62df\u5408\u8bad\u7ec3\u96c6\u3002\u4f46\u662f\uff0c\u8003\u8651\u5230\u8bad\u7ec3\u96c6\u548c\u6d4b\u8bd5\u96c6\u7cbe\u786e\u5ea6\u4e4b\u95f4\u7684\u5dee\u5f02\uff0c\u60a8\u53ef\u4ee5\u5c1d\u8bd5\u6dfb\u52a0 L2 \u6216 dropout \u6b63\u5219\u5316\u53bb\u51cf\u5c11\u8fc7\u62df\u5408\u3002\n",
      "- \u628a\u4f1a\u8bdd\u89c6\u4e3a\u4e00\u7ec4\u8bad\u7ec3\u6a21\u578b\u7684\u4ee3\u7801\u3002\u6bcf\u6b21\u5728\u5c0f\u6279\u91cf\u96c6\u4e0a\u8fd0\u884c\u4f1a\u8bdd\u65f6\uff0c\u90fd\u4f1a\u8bad\u7ec3\u53c2\u6570\u3002In total you have run the session a large number of times (1500 epochs) until you obtained well trained parameters."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "### 2.7 - Test with your own image (optional / ungraded exercise)\n",
      "\n",
      "\u795d\u8d3a\u60a8\u5b8c\u6210\u8fd9\u9879\u4efb\u52a1\u3002\u60a8\u73b0\u5728\u53ef\u4ee5\u62cd\u6444\u60a8\u7684\u624b\u7684\u7167\u7247\uff0c\u5e76\u67e5\u770b\u6a21\u578b\u7684\u8f93\u51fa\uff1a\n",
      "    1. Click on \"File\" in the upper bar of this notebook, then click \"Open\" to go on your Coursera Hub.\n",
      "    2. Add your image to this Jupyter Notebook's directory, in the \"images\" folder\n",
      "    3. Write your image's name in the following code\n",
      "    4. Run the code and check if the algorithm is right!"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": true,
     "input": [
      "import scipy\n",
      "from PIL import Image\n",
      "from scipy import ndimage\n",
      "\n",
      "## START CODE HERE ## (PUT YOUR IMAGE NAME) \n",
      "my_image = \"thumbs_up.jpg\"\n",
      "## END CODE HERE ##\n",
      "\n",
      "# We preprocess your image to fit your algorithm.\n",
      "fname = \"images/\" + my_image\n",
      "image = np.array(ndimage.imread(fname, flatten=False))\n",
      "my_image = scipy.misc.imresize(image, size=(64,64)).reshape((1, 64*64*3)).T\n",
      "my_image_prediction = predict(my_image, parameters)\n",
      "\n",
      "plt.imshow(image)\n",
      "print(\"Your algorithm predicts: y = \" + str(np.squeeze(my_image_prediction)))"
     ],
     "language": "python",
     "metadata": {
      "scrolled": true
     },
     "outputs": [],
     "prompt_number": null
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "You indeed deserved a \"thumbs-up\" although as you can see the algorithm seems to classify it incorrectly. \u539f\u56e0\u662f\u8bad\u7ec3\u96c6\u4e0d\u5305\u542b\u4efb\u4f55\u201c\u7ad6\u8d77\u5927\u62c7\u6307\u201d\u7684\u56fe\u50cf\uff0c\u6240\u4ee5\u6a21\u578b\u4e0d\u77e5\u9053\u8be5\u5982\u4f55\u5904\u7406!\u6211\u4eec\u79f0\u8fd9\u4e2a\u4e3a \"mismatched data distribution\"\uff0c \u662f\u4e0b\u4e00\u4e2a\u8bfe\u7a0b \"Structuring Machine Learning Projects\" \u8981\u8bb2\u89e3\u7684\u5185\u5bb9\u4e4b\u4e00."
     ]
    },
    {
     "cell_type": "markdown",
     "metadata": {
      "collapsed": true
     },
     "source": [
      "<font color='blue'>\n",
      "**What you should remember**:\n",
      "- \u5728\u6df1\u5ea6\u5b66\u4e60\u4e2d Tensorflow \u662f\u4e00\u4e2a\u7f16\u7a0b\u6846\u67b6\n",
      "- tensorflow \u6709\u4e24\u4e2a\u5f88\u91cd\u8981\u7684\u5bf9\u8c61\u7c7b Tensors \u548c Operators. \n",
      "- \u5f53\u4f60\u7528 tensorflow \u7f16\u7801\u65f6\uff0c\u4f60\u7684\u7f16\u7801\u6b65\u9aa4\u5fc5\u987b\u662f\uff1a\n",
      "    - \u521b\u5efa\u4e00\u4e2a\u5305\u542b Tensors (Variables, Placeholders ...) \u548c Operations (tf.matmul, tf.add, ...)\u7684\u8ba1\u7b97\u56fe\n",
      "    - \u521b\u5efa\u4e00\u4e2a\u4f1a\u8bdd\n",
      "    - \u521d\u59cb\u5316\u4f1a\u8bdd\n",
      "    - \u8fd0\u884c\u4f1a\u8bdd\u4ee5\u6267\u884c\u8ba1\u7b97\u56fe\n",
      "- \u60a8\u80fd\u591a\u6b21\u6267\u884c\u8ba1\u7b97\u56fe\u5c31\u50cf\u4f60\u5728 model\uff08\uff09\u4e2d\u770b\u5230\u7684\u90a3\u6837\n",
      "- \u5f53\u8fd0\u884c\u4f1a\u8bdd\u91cc\u7684 \"optimizer\" \u5bf9\u8c61\u65f6\uff0c\u53cd\u5411\u4f20\u64ad\u548c\u4f18\u5316\u4f1a\u81ea\u52a8\u5b8c\u6210\u3002"
     ]
    }
   ],
   "metadata": {}
  }
 ]
}